You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/12/10 17:37:54 UTC
svn commit: r1419570 [13/14] - in /lucene/dev/branches/lucene4547: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/
dev-tools/idea/solr/contrib/dataimporthandler/ dev-tools/maven/
dev-tools/maven/luce...
Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml Mon Dec 10 16:36:47 2012
@@ -64,6 +64,93 @@
<str name="queryAnalyzerFieldType">phrase_suggest</str>
</searchComponent>
+ <!-- FuzzyLookup suggest component (default)-->
+ <searchComponent class="solr.SpellCheckComponent" name="fuzzy_suggest_analyzing">
+ <lst name="spellchecker">
+ <str name="name">fuzzy_suggest_analyzing</str>
+ <str name="classname">org.apache.solr.spelling.suggest.Suggester</str>
+ <str name="lookupImpl">org.apache.solr.spelling.suggest.fst.FuzzyLookupFactory</str>
+ <str name="storeDir">fuzzy_suggest_analyzing</str>
+ <str name="buildOnCommit">false</str>
+
+ <!-- Suggester properties -->
+ <bool name="exactMatchFirst">true</bool>
+ <str name="suggestAnalyzerFieldType">text</str>
+ <bool name="preserveSep">false</bool>
+
+ <str name="sourceLocation">fuzzysuggest.txt</str>
+ </lst>
+
+ <!-- specify a fieldtype using keywordtokenizer + lowercase + cleanup -->
+ <str name="queryAnalyzerFieldType">phrase_suggest</str>
+ </searchComponent>
+
+ <!-- FuzzyLookup suggest component with max edit 2-->
+ <searchComponent class="solr.SpellCheckComponent" name="fuzzy_suggest_analyzing_with_max_edit_2">
+ <lst name="spellchecker">
+ <str name="name">fuzzy_suggest_analyzing_with_max_edit_2</str>
+ <str name="classname">org.apache.solr.spelling.suggest.Suggester</str>
+ <str name="lookupImpl">org.apache.solr.spelling.suggest.fst.FuzzyLookupFactory</str>
+ <str name="storeDir">fuzzy_suggest_analyzing_with_max_edit_2</str>
+ <str name="buildOnCommit">false</str>
+
+ <!-- Suggester properties -->
+ <bool name="exactMatchFirst">true</bool>
+ <str name="suggestAnalyzerFieldType">text</str>
+ <bool name="preserveSep">false</bool>
+ <int name="maxEdits">2</int>
+
+ <str name="sourceLocation">fuzzysuggest.txt</str>
+ </lst>
+
+ <!-- specify a fieldtype using keywordtokenizer + lowercase + cleanup -->
+ <str name="queryAnalyzerFieldType">phrase_suggest</str>
+ </searchComponent>
+
+ <!-- FuzzyLookup suggest component with 4 non_fuzzy_prefix -->
+ <searchComponent class="solr.SpellCheckComponent" name="fuzzy_suggest_analyzing_with_non_fuzzy_prefix_4">
+ <lst name="spellchecker">
+ <str name="name">fuzzy_suggest_analyzing_with_non_fuzzy_prefix_4</str>
+ <str name="classname">org.apache.solr.spelling.suggest.Suggester</str>
+ <str name="lookupImpl">org.apache.solr.spelling.suggest.fst.FuzzyLookupFactory</str>
+ <str name="storeDir">fuzzy_suggest_analyzing_with_non_fuzzy_prefix_4</str>
+ <str name="buildOnCommit">false</str>
+
+ <!-- Suggester properties -->
+ <bool name="exactMatchFirst">true</bool>
+ <str name="suggestAnalyzerFieldType">text</str>
+ <bool name="preserveSep">false</bool>
+ <int name="nonFuzzyPrefix">4</int>
+
+ <str name="sourceLocation">fuzzysuggest.txt</str>
+ </lst>
+
+ <!-- specify a fieldtype using keywordtokenizer + lowercase + cleanup -->
+ <str name="queryAnalyzerFieldType">phrase_suggest</str>
+ </searchComponent>
+
+ <!-- FuzzyLookup suggest component with 2 min_fuzzy_length -->
+ <searchComponent class="solr.SpellCheckComponent" name="fuzzy_suggest_analyzing_with_min_fuzzy_length_2">
+ <lst name="spellchecker">
+ <str name="name">fuzzy_suggest_analyzing_with_min_fuzzy_length_2</str>
+ <str name="classname">org.apache.solr.spelling.suggest.Suggester</str>
+ <str name="lookupImpl">org.apache.solr.spelling.suggest.fst.FuzzyLookupFactory</str>
+ <str name="storeDir">fuzzy_suggest_analyzing_with_min_fuzzy_length_2</str>
+ <str name="buildOnCommit">false</str>
+
+ <!-- Suggester properties -->
+ <bool name="exactMatchFirst">true</bool>
+ <str name="suggestAnalyzerFieldType">text</str>
+ <bool name="preserveSep">false</bool>
+ <int name="minFuzzyLength">2</int>
+
+ <str name="sourceLocation">fuzzysuggest.txt</str>
+ </lst>
+
+ <!-- specify a fieldtype using keywordtokenizer + lowercase + cleanup -->
+ <str name="queryAnalyzerFieldType">phrase_suggest</str>
+ </searchComponent>
+
<!-- is this thing just configured globally or wtf is going on here?! -->
<queryConverter name="queryConverter" class="org.apache.solr.spelling.SuggestQueryConverter"/>
@@ -95,4 +182,59 @@
</arr>
</requestHandler>
+ <!-- Fuzzy analyzing handler with 1 max edit -->
+ <requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing">
+ <lst name="defaults">
+ <str name="spellcheck">true</str>
+ <str name="spellcheck.dictionary">fuzzy_suggest_analyzing</str>
+ <str name="spellcheck.collate">false</str>
+ <!-- NOTE: if this is false, results are alpha-ordered, not by weight! -->
+ <str name="spellcheck.onlyMorePopular">true</str>
+ </lst>
+ <arr name="components">
+ <str>fuzzy_suggest_analyzing</str>
+ </arr>
+ </requestHandler>
+
+ <!-- Fuzzy analyzing handler with 2 max edit -->
+ <requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing_with_max_edit_2">
+ <lst name="defaults">
+ <str name="spellcheck">true</str>
+ <str name="spellcheck.dictionary">fuzzy_suggest_analyzing_with_max_edit_2</str>
+ <str name="spellcheck.collate">false</str>
+ <!-- NOTE: if this is false, results are alpha-ordered, not by weight! -->
+ <str name="spellcheck.onlyMorePopular">true</str>
+ </lst>
+ <arr name="components">
+ <str>fuzzy_suggest_analyzing_with_max_edit_2</str>
+ </arr>
+ </requestHandler>
+
+ <!-- Fuzzy analyzing handler with 4 non_fuzzy_prefix -->
+ <requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing_with_non_fuzzy_prefix_4">
+ <lst name="defaults">
+ <str name="spellcheck">true</str>
+ <str name="spellcheck.dictionary">fuzzy_suggest_analyzing_with_non_fuzzy_prefix_4</str>
+ <str name="spellcheck.collate">false</str>
+ <!-- NOTE: if this is false, results are alpha-ordered, not by weight! -->
+ <str name="spellcheck.onlyMorePopular">true</str>
+ </lst>
+ <arr name="components">
+ <str>fuzzy_suggest_analyzing_with_non_fuzzy_prefix_4</str>
+ </arr>
+ </requestHandler>
+
+ <!-- Fuzzy analyzing handler with 2 min_fuzzy_length -->
+ <requestHandler class="org.apache.solr.handler.component.SearchHandler" name="/fuzzy_suggest_analyzing_with_min_fuzzy_length_2">
+ <lst name="defaults">
+ <str name="spellcheck">true</str>
+ <str name="spellcheck.dictionary">fuzzy_suggest_analyzing_with_min_fuzzy_length_2</str>
+ <str name="spellcheck.collate">false</str>
+ <!-- NOTE: if this is false, results are alpha-ordered, not by weight! -->
+ <str name="spellcheck.onlyMorePopular">true</str>
+ </lst>
+ <arr name="components">
+ <str>fuzzy_suggest_analyzing_with_min_fuzzy_length_2</str>
+ </arr>
+ </requestHandler>
</config>
Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Mon Dec 10 16:36:47 2012
@@ -90,7 +90,7 @@
-->
<updateLog enable="${enable.update.log:false}">
- <str name="dir">${solr.data.dir:}</str>
+ <str name="dir">${solr.ulog.dir:}</str>
</updateLog>
</updateHandler>
Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml Mon Dec 10 16:36:47 2012
@@ -22,14 +22,14 @@
persistent: Save changes made via the API to this file
sharedLib: path to a lib directory that will be shared across all cores
-->
-<solr persistent="false">
+<solr persistent="${solr.xml.persist:false}">
<!--
adminPath: RequestHandler path to manage cores.
If 'null' (or absent), cores will not be manageable via request handler
-->
<cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}"
- hostContext="solr" zkClientTimeout="5000" numShards="${numShards:3}" shareSchema="${shareSchema:false}">
+ hostContext="solr" zkClientTimeout="8000" numShards="${numShards:3}" shareSchema="${shareSchema:false}">
<core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"/>
</cores>
</solr>
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java Mon Dec 10 16:36:47 2012
@@ -179,6 +179,19 @@ public class TestDistributedGrouping ext
query("q", "*:*", "fq", s1 + ":a", "fl", "id," + i1, "group", "true", "group.field", i1, "sort", i1 + " asc, id asc", "group.ngroups", "true");
query("q", "*:*", "fq", s1 + ":a", "rows", 0, "fl", "id," + i1, "group", "true", "group.field", i1, "sort", i1 + " asc, id asc", "group.ngroups", "true");
+ // SOLR-3960 - include a postfilter
+ for (String facet : new String[] { "false", "true"}) {
+ for (String fcache : new String[] { "", " cache=false cost=200"}) {
+ query("q", "*:*", "rows", 100, "fl", "id," + i1,
+ "group.limit", 10, "sort", i1 + " asc, id asc",
+ "group", "true", "group.field", i1,
+ "fq", "{!frange l=50 "+fcache+"}"+tlong,
+ "facet.field", t1,
+ "facet", facet
+ );
+ }
+ }
+
ModifiableSolrParams params = new ModifiableSolrParams();
Object[] q = {"q", "*:*", "fq", s1 + ":a", "rows", 1, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "group.ngroups", "true"};
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Mon Dec 10 16:36:47 2012
@@ -60,6 +60,7 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -126,6 +127,7 @@ public class BasicDistributedZkTest exte
public void setUp() throws Exception {
super.setUp();
System.setProperty("numShards", Integer.toString(sliceCount));
+ System.setProperty("solr.xml.persist", "true");
}
@@ -341,12 +343,83 @@ public class BasicDistributedZkTest exte
testCollectionsAPI();
testCoreUnloadAndLeaders();
testUnloadLotsOfCores();
+ testStopAndStartCoresInOneInstance();
+ testUnloadShardAndCollection();
// Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
}
}
+ private void testUnloadShardAndCollection() throws Exception{
+ // create one leader and one replica
+
+ Create createCmd = new Create();
+ createCmd.setCoreName("test_unload_shard_and_collection_1");
+ String collection = "test_unload_shard_and_collection";
+ createCmd.setCollection(collection);
+ String coreDataDir = dataDir.getAbsolutePath() + File.separator
+ + System.currentTimeMillis() + collection + "1";
+ createCmd.setDataDir(coreDataDir);
+ createCmd.setNumShards(2);
+
+ SolrServer client = clients.get(0);
+ String url1 = getBaseUrl(client);
+ HttpSolrServer server = new HttpSolrServer(url1);
+
+ server.request(createCmd);
+
+ createCmd = new Create();
+ createCmd.setCoreName("test_unload_shard_and_collection_2");
+ collection = "test_unload_shard_and_collection";
+ createCmd.setCollection(collection);
+ coreDataDir = dataDir.getAbsolutePath() + File.separator
+ + System.currentTimeMillis() + collection + "2";
+ createCmd.setDataDir(coreDataDir);
+
+ server.request(createCmd);
+
+ // now unload one of the two
+ Unload unloadCmd = new Unload(false);
+ unloadCmd.setCoreName("test_unload_shard_and_collection_2");
+ server.request(unloadCmd);
+
+ // there should be only one shard
+ Slice shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
+ long timeoutAt = System.currentTimeMillis() + 30000;
+ while (shard2 != null) {
+ if (System.currentTimeMillis() > timeoutAt) {
+ printLayout();
+ fail("Still found shard");
+ }
+
+ Thread.sleep(50);
+ shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
+ }
+
+ Slice shard1 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard1");
+ assertNotNull(shard1);
+ assertTrue(getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection));
+
+ // now unload one of the other
+ unloadCmd = new Unload(false);
+ unloadCmd.setCoreName("test_unload_shard_and_collection_1");
+ server.request(unloadCmd);
+
+ //printLayout();
+ // the collection should be gone
+ timeoutAt = System.currentTimeMillis() + 30000;
+ while (getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection)) {
+ if (System.currentTimeMillis() > timeoutAt) {
+ printLayout();
+ fail("Still found collection");
+ }
+
+ Thread.sleep(50);
+ }
+
+ }
+
/**
* @throws Exception on any problem
*/
@@ -364,11 +437,11 @@ public class BasicDistributedZkTest exte
createCmd.setDataDir(core1DataDir);
server.request(createCmd);
- ZkStateReader zkStateReader = solrj.getZkStateReader();
+ ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
zkStateReader.updateClusterState(true);
- int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").size();
+ int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
assertEquals(1, slices);
client = clients.get(1);
@@ -383,7 +456,7 @@ public class BasicDistributedZkTest exte
server.request(createCmd);
zkStateReader.updateClusterState(true);
- slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").size();
+ slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
assertEquals(1, slices);
waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
@@ -448,7 +521,7 @@ public class BasicDistributedZkTest exte
// Thread.currentThread().sleep(500);
// printLayout();
- int tries = 20;
+ int tries = 50;
while (leaderProps.getCoreUrl().equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
Thread.sleep(100);
if (tries-- == 0) {
@@ -550,54 +623,96 @@ public class BasicDistributedZkTest exte
5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
int cnt = atLeast(6);
- for (int i = 0; i < cnt; i++) {
- final int freezeI = i;
+
+ // create the 6 cores
+ createCores(server, executor, "multiunload", 2, cnt);
+
+ executor.shutdown();
+ executor.awaitTermination(120, TimeUnit.SECONDS);
+ executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
+ TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ new DefaultSolrThreadFactory("testExecutor"));
+ for (int j = 0; j < cnt; j++) {
+ final int freezeJ = j;
executor.execute(new Runnable() {
-
@Override
public void run() {
- Create createCmd = new Create();
- createCmd.setCoreName("multiunload" + freezeI);
- createCmd.setCollection("multiunload");
- String core3dataDir = dataDir.getAbsolutePath() + File.separator
- + System.currentTimeMillis() + "unloadcollection" + "_3n" + freezeI;
- createCmd.setDataDir(core3dataDir);
+ Unload unloadCmd = new Unload(true);
+ unloadCmd.setCoreName("multiunload" + freezeJ);
try {
- server.request(createCmd);
+ server.request(unloadCmd);
} catch (SolrServerException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
-
});
+ Thread.sleep(random().nextInt(50));
}
executor.shutdown();
executor.awaitTermination(120, TimeUnit.SECONDS);
- executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
- TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ }
+
+ private void testStopAndStartCoresInOneInstance() throws Exception {
+ SolrServer client = clients.get(0);
+ String url3 = getBaseUrl(client);
+ final HttpSolrServer server = new HttpSolrServer(url3);
+
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
+ 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
- for (int j = 0; j < cnt; j++) {
- final int freezeJ = j;
+ int cnt = 3;
+
+ // create the cores
+ createCores(server, executor, "multiunload2", 1, cnt);
+
+ executor.shutdown();
+ executor.awaitTermination(120, TimeUnit.SECONDS);
+
+ ChaosMonkey.stop(cloudJettys.get(0).jetty);
+ printLayout();
+
+ Thread.sleep(5000);
+ ChaosMonkey.start(cloudJettys.get(0).jetty);
+ cloudClient.getZkStateReader().updateClusterState(true);
+ try {
+ cloudClient.getZkStateReader().getLeaderProps("multiunload2", "shard1", 30000);
+ } catch (SolrException e) {
+ printLayout();
+ throw e;
+ }
+
+ printLayout();
+
+ }
+
+ private void createCores(final HttpSolrServer server,
+ ThreadPoolExecutor executor, final String collection, final int numShards, int cnt) {
+ for (int i = 0; i < cnt; i++) {
+ final int freezeI = i;
executor.execute(new Runnable() {
+
@Override
public void run() {
- Unload unloadCmd = new Unload(true);
- unloadCmd.setCoreName("multiunload" + freezeJ);
+ Create createCmd = new Create();
+ createCmd.setCoreName(collection + freezeI);
+ createCmd.setCollection(collection);
+ String core3dataDir = dataDir.getAbsolutePath() + File.separator
+ + System.currentTimeMillis() + collection + "_3n" + freezeI;
+ createCmd.setDataDir(core3dataDir);
+ createCmd.setNumShards(numShards);
try {
- server.request(unloadCmd);
+ server.request(createCmd);
} catch (SolrServerException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
+
});
- Thread.sleep(random().nextInt(50));
}
- executor.shutdown();
- executor.awaitTermination(120, TimeUnit.SECONDS);
}
@@ -623,16 +738,17 @@ public class BasicDistributedZkTest exte
int cnt = atLeast(3);
for (int i = 0; i < cnt; i++) {
- createCollection(collectionInfos, i,
- _TestUtil.nextInt(random(), 0, shardCount) + 1,
- _TestUtil.nextInt(random(), 0, 5) + 1);
+ int numShards = _TestUtil.nextInt(random(), 0, shardCount) + 1;
+ int numReplica = _TestUtil.nextInt(random(), 0, 3) + 1;
+ int maxShardsPerNode = (((numShards * (numReplica + 1)) / getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size())) + 1;
+ createCollection(collectionInfos, i, numShards, numReplica, maxShardsPerNode);
}
Set<Entry<String,List<Integer>>> collectionInfosEntrySet = collectionInfos.entrySet();
for (Entry<String,List<Integer>> entry : collectionInfosEntrySet) {
String collection = entry.getKey();
List<Integer> list = entry.getValue();
- checkForCollection(collection, list.get(0));
+ checkForCollection(collection, list);
String url = getUrlFromZk(collection);
@@ -641,7 +757,7 @@ public class BasicDistributedZkTest exte
// poll for a second - it can take a moment before we are ready to serve
waitForNon403or404or503(collectionClient);
}
- ZkStateReader zkStateReader = solrj.getZkStateReader();
+ ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
for (int j = 0; j < cnt; j++) {
waitForRecoveriesToFinish("awholynewcollection_" + j, zkStateReader, false);
}
@@ -736,7 +852,10 @@ public class BasicDistributedZkTest exte
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
- checkForCollection(collectionName, 1);
+ List<Integer> list = new ArrayList<Integer> (2);
+ list.add(1);
+ list.add(1);
+ checkForCollection(collectionName, list);
url = getUrlFromZk(collectionName);
@@ -748,21 +867,34 @@ public class BasicDistributedZkTest exte
for (int j = 0; j < cnt; j++) {
waitForRecoveriesToFinish(collectionName, zkStateReader, false);
}
+
+ // test maxShardsPerNode
+ int liveNodes = getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size();
+ int numShards = (liveNodes/2) + 1;
+ int numReplica = 1;
+ int maxShardsPerNode = 1;
+ collectionInfos = new HashMap<String,List<Integer>>();
+ createCollection(collectionInfos, cnt, numShards, numReplica, maxShardsPerNode);
+
+ // TODO: enable this check after removing the 60 second wait in it
+ //checkCollectionIsNotCreated(collectionInfos.keySet().iterator().next());
}
protected void createCollection(Map<String,List<Integer>> collectionInfos,
- int i, int numShards, int numReplicas) throws SolrServerException, IOException {
+ int i, int numShards, int numReplica, int maxShardsPerNode) throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
- params.set("numShards", numShards);
- params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
+ params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
+ params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplica);
+ params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
String collectionName = "awholynewcollection_" + i;
int clientIndex = random().nextInt(2);
List<Integer> list = new ArrayList<Integer>();
list.add(numShards);
- list.add(numReplicas);
+ list.add(numReplica);
+ list.add(maxShardsPerNode);
collectionInfos.put(collectionName, list);
params.set("name", collectionName);
SolrRequest request = new QueryRequest(params);
@@ -808,10 +940,10 @@ public class BasicDistributedZkTest exte
private void collectStartTimes(String collectionName,
Map<String,Long> urlToTime) throws SolrServerException, IOException {
- Map<String,Map<String,Slice>> collections = solrj.getZkStateReader()
+ Map<String,DocCollection> collections = getCommonCloudSolrServer().getZkStateReader()
.getClusterState().getCollectionStates();
if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName);
+ Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
Iterator<Entry<String,Slice>> it = slices.entrySet().iterator();
while (it.hasNext()) {
@@ -836,8 +968,8 @@ public class BasicDistributedZkTest exte
}
private String getUrlFromZk(String collection) {
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
- Map<String,Slice> slices = clusterState.getCollectionStates().get(collection);
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+ Map<String,Slice> slices = clusterState.getCollectionStates().get(collection).getSlicesMap();
if (slices == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
@@ -850,7 +982,7 @@ public class BasicDistributedZkTest exte
for (Map.Entry<String,Replica> shardEntry : shardEntries) {
final ZkNodeProps node = shardEntry.getValue();
if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
- return new ZkCoreNodeProps(node).getCoreUrl();
+ return ZkCoreNodeProps.getCoreUrl(node.getStr(ZkStateReader.BASE_URL_PROP), collection); //new ZkCoreNodeProps(node).getCoreUrl();
}
}
}
@@ -859,7 +991,7 @@ public class BasicDistributedZkTest exte
}
private ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
ZkNodeProps leader = clusterState.getLeader(collection, slice);
if (leader == null) {
throw new RuntimeException("Could not find leader:" + collection + " " + slice);
@@ -893,33 +1025,60 @@ public class BasicDistributedZkTest exte
fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
}
- private void checkForCollection(String collectionName, int expectedSlices)
- throws Exception {
- // check for an expectedSlices new collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 60000;
- boolean found = false;
- boolean sliceMatch = false;
- while (System.currentTimeMillis() < timeoutAt) {
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
- Map<String,Map<String,Slice>> collections = clusterState
+ private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList) {
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+
+ int expectedSlices = numShardsNumReplicaList.get(0);
+ // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
+ int expectedShardsPerSlice = numShardsNumReplicaList.get(1) + 1;
+ int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
+
+ Map<String,DocCollection> collections = clusterState
.getCollectionStates();
if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName);
+ Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
// did we find expectedSlices slices/shards?
- if (slices.size() == expectedSlices) {
- sliceMatch = true;
-
+ if (slices.size() != expectedSlices) {
+ return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
+ }
+ int totalShards = 0;
+ for (String sliceName : slices.keySet()) {
+ totalShards += slices.get(sliceName).getReplicas().size();
+ }
+ if (totalShards != expectedTotalShards) {
+ return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
}
- found = true;
+ return null;
+ } else {
+ return "Could not find new collection " + collectionName;
+ }
+ }
+
+ private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList)
+ throws Exception {
+ // check for an expectedSlices new collection - we poll the state
+ long timeoutAt = System.currentTimeMillis() + 120000;
+ boolean success = false;
+ String checkResult = "Didnt get to perform a single check";
+ while (System.currentTimeMillis() < timeoutAt) {
+ checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList);
+ if (checkResult == null) {
+ success = true;
break;
}
Thread.sleep(500);
}
- if (!found) {
- if (!sliceMatch) {
- fail("Could not find new " + expectedSlices + " slice collection called " + collectionName);
+ if (!success) {
+ super.printLayout();
+ fail(checkResult);
}
}
+
+ private void checkCollectionIsNotCreated(String collectionName)
+ throws Exception {
+ // TODO: this method not called because of below sleep
+ Thread.sleep(60000);
+ assertFalse(collectionName + " not supposed to exist", getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collectionName));
}
private void checkForMissingCollection(String collectionName)
@@ -928,9 +1087,9 @@ public class BasicDistributedZkTest exte
long timeoutAt = System.currentTimeMillis() + 15000;
boolean found = true;
while (System.currentTimeMillis() < timeoutAt) {
- solrj.getZkStateReader().updateClusterState(true);
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
- Map<String,Map<String,Slice>> collections = clusterState
+ getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+ Map<String,DocCollection> collections = clusterState
.getCollectionStates();
if (!collections.containsKey(collectionName)) {
found = false;
@@ -993,6 +1152,7 @@ public class BasicDistributedZkTest exte
// cloud level test mainly needed just to make sure that versions and errors are propagated correctly
private void doOptimisticLockingAndUpdating() throws Exception {
+ log.info("### STARTING doOptimisticLockingAndUpdating");
printLayout();
SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);
@@ -1033,6 +1193,7 @@ public class BasicDistributedZkTest exte
private void testNumberOfCommitsWithCommitAfterAdd()
throws SolrServerException, IOException {
+ log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd");
long startCommits = getNumCommits((HttpSolrServer) clients.get(0));
ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update");
@@ -1064,6 +1225,7 @@ public class BasicDistributedZkTest exte
}
private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
+ log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");
System.clearProperty("numShards");
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
@@ -1090,9 +1252,9 @@ public class BasicDistributedZkTest exte
// no one should be recovering
- waitForRecoveriesToFinish(oneInstanceCollection2, solrj.getZkStateReader(), false, true);
+ waitForRecoveriesToFinish(oneInstanceCollection2, getCommonCloudSolrServer().getZkStateReader(), false, true);
- assertAllActive(oneInstanceCollection2, solrj.getZkStateReader());
+ assertAllActive(oneInstanceCollection2, getCommonCloudSolrServer().getZkStateReader());
//printLayout();
@@ -1113,7 +1275,7 @@ public class BasicDistributedZkTest exte
query.set("collection", oneInstanceCollection2);
query.set("distrib", true);
- long allDocs = solrj.query(query).getResults().getNumFound();
+ long allDocs = getCommonCloudSolrServer().query(query).getResults().getNumFound();
// System.out.println("1:" + oneDocs);
// System.out.println("2:" + twoDocs);
@@ -1127,15 +1289,15 @@ public class BasicDistributedZkTest exte
assertEquals(3, allDocs);
// we added a role of none on these creates - check for it
- ZkStateReader zkStateReader = solrj.getZkStateReader();
+ ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
zkStateReader.updateClusterState(true);
- Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(oneInstanceCollection2);
+ Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(oneInstanceCollection2);
assertNotNull(slices);
String roles = slices.get("slice1").getReplicasMap().values().iterator().next().getStr(ZkStateReader.ROLES_PROP);
assertEquals("none", roles);
- ZkCoreNodeProps props = new ZkCoreNodeProps(solrj.getZkStateReader().getClusterState().getLeader(oneInstanceCollection2, "slice1"));
+ ZkCoreNodeProps props = new ZkCoreNodeProps(getCommonCloudSolrServer().getZkStateReader().getClusterState().getLeader(oneInstanceCollection2, "slice1"));
// now test that unloading a core gets us a new leader
HttpSolrServer server = new HttpSolrServer(baseUrl);
@@ -1157,6 +1319,7 @@ public class BasicDistributedZkTest exte
}
private void testSearchByCollectionName() throws SolrServerException {
+ log.info("### STARTING testSearchByCollectionName");
SolrServer client = clients.get(0);
final String baseUrl = ((HttpSolrServer) client).getBaseURL().substring(
0,
@@ -1172,6 +1335,7 @@ public class BasicDistributedZkTest exte
}
private void testANewCollectionInOneInstance() throws Exception {
+ log.info("### STARTING testANewCollectionInOneInstance");
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
otherCollectionClients.put(oneInstanceCollection , collectionClients);
@@ -1196,8 +1360,8 @@ public class BasicDistributedZkTest exte
SolrServer client3 = createNewSolrServer(oneInstanceCollection + "3", baseUrl);
SolrServer client4 = createNewSolrServer(oneInstanceCollection + "4", baseUrl);
- waitForRecoveriesToFinish(oneInstanceCollection, solrj.getZkStateReader(), false);
- assertAllActive(oneInstanceCollection, solrj.getZkStateReader());
+ waitForRecoveriesToFinish(oneInstanceCollection, getCommonCloudSolrServer().getZkStateReader(), false);
+ assertAllActive(oneInstanceCollection, getCommonCloudSolrServer().getZkStateReader());
client2.add(getDoc(id, "1"));
client3.add(getDoc(id, "2"));
@@ -1213,7 +1377,7 @@ public class BasicDistributedZkTest exte
query.set("collection", oneInstanceCollection);
query.set("distrib", true);
- long allDocs = solrj.query(query).getResults().getNumFound();
+ long allDocs = getCommonCloudSolrServer().query(query).getResults().getNumFound();
// System.out.println("1:" + oneDocs);
// System.out.println("2:" + twoDocs);
@@ -1266,6 +1430,7 @@ public class BasicDistributedZkTest exte
}
private void testMultipleCollections() throws Exception {
+ log.info("### STARTING testMultipleCollections");
// create another 2 collections and search across them
createNewCollection("collection2");
createNewCollection("collection3");
@@ -1280,21 +1445,21 @@ public class BasicDistributedZkTest exte
indexDoc("collection2", getDoc(id, "10000000"));
indexDoc("collection2", getDoc(id, "10000001"));
indexDoc("collection2", getDoc(id, "10000003"));
- solrj.setDefaultCollection("collection2");
- solrj.add(getDoc(id, "10000004"));
- solrj.setDefaultCollection(null);
+ getCommonCloudSolrServer().setDefaultCollection("collection2");
+ getCommonCloudSolrServer().add(getDoc(id, "10000004"));
+ getCommonCloudSolrServer().setDefaultCollection(null);
indexDoc("collection3", getDoc(id, "20000000"));
indexDoc("collection3", getDoc(id, "20000001"));
- solrj.setDefaultCollection("collection3");
- solrj.add(getDoc(id, "10000005"));
- solrj.setDefaultCollection(null);
+ getCommonCloudSolrServer().setDefaultCollection("collection3");
+ getCommonCloudSolrServer().add(getDoc(id, "10000005"));
+ getCommonCloudSolrServer().setDefaultCollection(null);
otherCollectionClients.get("collection2").get(0).commit();
otherCollectionClients.get("collection3").get(0).commit();
- solrj.setDefaultCollection("collection1");
- long collection1Docs = solrj.query(new SolrQuery("*:*")).getResults()
+ getCommonCloudSolrServer().setDefaultCollection("collection1");
+ long collection1Docs = getCommonCloudSolrServer().query(new SolrQuery("*:*")).getResults()
.getNumFound();
long collection2Docs = otherCollectionClients.get("collection2").get(0)
@@ -1315,19 +1480,19 @@ public class BasicDistributedZkTest exte
assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
// try to search multiple with cloud client
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
query.set("collection", "collection2,collection3");
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection2Docs + collection3Docs, found);
query.set("collection", "collection3");
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection3Docs, found);
query.remove("collection");
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection1Docs, found);
assertEquals(collection3Docs, collection2Docs - 1);
@@ -1403,42 +1568,44 @@ public class BasicDistributedZkTest exte
}
}
- volatile CloudSolrServer solrj;
-
- @Override
- protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
-
- if (r.nextBoolean())
- return super.queryServer(params);
-
- // use the distributed solrj client
- if (solrj == null) {
+ volatile CloudSolrServer commondCloudSolrServer;
+ private CloudSolrServer getCommonCloudSolrServer() {
+ if (commondCloudSolrServer == null) {
synchronized(this) {
try {
- CloudSolrServer server = new CloudSolrServer(zkServer.getZkAddress());
- server.setDefaultCollection(DEFAULT_COLLECTION);
- solrj = server;
+ commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
+ commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
+ commondCloudSolrServer.connect();
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
}
}
+ return commondCloudSolrServer;
+ }
+
+ @Override
+ protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
+
+ if (r.nextBoolean())
+ return super.queryServer(params);
if (r.nextBoolean())
params.set("collection",DEFAULT_COLLECTION);
- QueryResponse rsp = solrj.query(params);
+ QueryResponse rsp = getCommonCloudSolrServer().query(params);
return rsp;
}
@Override
public void tearDown() throws Exception {
super.tearDown();
- if (solrj != null) {
- solrj.shutdown();
+ if (commondCloudSolrServer != null) {
+ commondCloudSolrServer.shutdown();
}
System.clearProperty("numShards");
System.clearProperty("zkHost");
+ System.clearProperty("solr.xml.persist");
// insurance
DirectUpdateHandler2.commitOnClose = true;
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java Mon Dec 10 16:36:47 2012
@@ -24,6 +24,9 @@ import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -32,7 +35,7 @@ import org.junit.Test;
public class ClusterStateTest extends SolrTestCaseJ4 {
@Test
public void testStoreAndRead() throws Exception {
- Map<String,Map<String,Slice>> collectionStates = new HashMap<String,Map<String,Slice>>();
+ Map<String,DocCollection> collectionStates = new HashMap<String,DocCollection>();
Set<String> liveNodes = new HashSet<String>();
liveNodes.add("node1");
liveNodes.add("node2");
@@ -49,12 +52,12 @@ public class ClusterStateTest extends So
slices.put("shard1", slice);
Slice slice2 = new Slice("shard2", sliceToProps, null);
slices.put("shard2", slice2);
- collectionStates.put("collection1", slices);
- collectionStates.put("collection2", slices);
+ collectionStates.put("collection1", new DocCollection("collection1", slices, null, DocRouter.DEFAULT));
+ collectionStates.put("collection2", new DocCollection("collection2", slices, null, DocRouter.DEFAULT));
ClusterState clusterState = new ClusterState(liveNodes, collectionStates);
byte[] bytes = ZkStateReader.toJSON(clusterState);
-
+ // System.out.println("#################### " + new String(bytes));
ClusterState loadedClusterState = ClusterState.load(null, bytes, liveNodes);
assertEquals("Provided liveNodes not used properly", 2, loadedClusterState
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java Mon Dec 10 16:36:47 2012
@@ -171,7 +171,7 @@ public class ClusterStateUpdateTest exte
Map<String,Slice> slices = null;
for (int i = 75; i > 0; i--) {
clusterState2 = zkController2.getClusterState();
- slices = clusterState2.getSlices("testcore");
+ slices = clusterState2.getSlicesMap("testcore");
if (slices != null && slices.containsKey("shard1")
&& slices.get("shard1").getReplicasMap().size() > 0) {
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java Mon Dec 10 16:36:47 2012
@@ -79,7 +79,7 @@ public class LeaderElectionIntegrationTe
ignoreException("No UpdateLog found - cannot sync");
ignoreException("No UpdateLog found - cannot recover");
- System.setProperty("zkClientTimeout", "3000");
+ System.setProperty("zkClientTimeout", "8000");
zkDir = dataDir.getAbsolutePath() + File.separator
+ "zookeeper" + System.currentTimeMillis() + "/server1/data";
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Mon Dec 10 16:36:47 2012
@@ -35,6 +35,7 @@ import javax.xml.parsers.ParserConfigura
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -148,7 +149,7 @@ public class OverseerTest extends SolrTe
}
private String getShardId(final String coreName) {
- Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(
+ Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(
collection);
if (slices != null) {
for (Slice slice : slices.values()) {
@@ -301,7 +302,7 @@ public class OverseerTest extends SolrTe
cloudStateSliceCount = 0;
reader.updateClusterState(true);
ClusterState state = reader.getClusterState();
- Map<String,Slice> slices = state.getSlices("collection1");
+ Map<String,Slice> slices = state.getSlicesMap("collection1");
for (String name : slices.keySet()) {
cloudStateSliceCount += slices.get(name).getReplicasMap().size();
}
@@ -712,8 +713,8 @@ public class OverseerTest extends SolrTe
ClusterState state = reader.getClusterState();
int numFound = 0;
- for (Map<String,Slice> collection : state.getCollectionStates().values()) {
- for (Slice slice : collection.values()) {
+ for (DocCollection collection : state.getCollectionStates().values()) {
+ for (Slice slice : collection.getSlices()) {
if (slice.getReplicasMap().get("node1_core1") != null) {
numFound++;
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java Mon Dec 10 16:36:47 2012
@@ -17,16 +17,26 @@ package org.apache.solr.cloud;
* the License.
*/
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.cloud.HashPartitioner;
-import org.apache.solr.common.cloud.HashPartitioner.Range;
+import org.apache.solr.common.cloud.CompositeIdRouter;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.DocRouter.Range;
+import org.apache.solr.common.cloud.PlainIdRouter;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.util.Hash;
+import org.apache.solr.common.util.StrUtils;
public class TestHashPartitioner extends SolrTestCaseJ4 {
public void testMapHashes() throws Exception {
- HashPartitioner hp = new HashPartitioner();
+ DocRouter hp = DocRouter.DEFAULT;
List<Range> ranges;
// make sure the partitioner uses the "natural" boundaries and doesn't suffer from an off-by-one
@@ -62,5 +72,167 @@ public class TestHashPartitioner extends
}
}
-
+
+ public int hash(String id) {
+ // our hashing is defined to be murmurhash3 on the UTF-8 bytes of the key.
+ return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
+ }
+
+ public void testHashCodes() throws Exception {
+ DocRouter router = DocRouter.getDocRouter(PlainIdRouter.NAME);
+ assertTrue(router instanceof PlainIdRouter);
+ DocCollection coll = createCollection(4, router);
+ doNormalIdHashing(coll);
+ }
+
+ public void doNormalIdHashing(DocCollection coll) throws Exception {
+ assertEquals(4, coll.getSlices().size());
+
+ doId(coll, "b", "shard1");
+ doId(coll, "c", "shard2");
+ doId(coll, "d", "shard3");
+ doId(coll, "e", "shard4");
+ }
+
+ public void doId(DocCollection coll, String id, String expectedShard) {
+ doIndex(coll, id, expectedShard);
+ doQuery(coll, id, expectedShard);
+ }
+
+ public void doIndex(DocCollection coll, String id, String expectedShard) {
+ DocRouter router = coll.getRouter();
+ Slice target = router.getTargetSlice(id, null, null, coll);
+ assertEquals(expectedShard, target.getName());
+ }
+
+ public void doQuery(DocCollection coll, String id, String expectedShards) {
+ DocRouter router = coll.getRouter();
+ Collection<Slice> slices = router.getSearchSlices(id, null, coll);
+
+ List<String> expectedShardStr = StrUtils.splitSmart(expectedShards, ",", true);
+
+ HashSet<String> expectedSet = new HashSet<String>(expectedShardStr);
+ HashSet<String> obtainedSet = new HashSet<String>();
+ for (Slice slice : slices) {
+ obtainedSet.add(slice.getName());
+ }
+
+ assertEquals(slices.size(), obtainedSet.size()); // make sure no repeated slices
+ assertEquals(expectedSet, obtainedSet);
+ }
+
+ public void testCompositeHashCodes() throws Exception {
+ DocRouter router = DocRouter.getDocRouter(CompositeIdRouter.NAME);
+ assertTrue(router instanceof CompositeIdRouter);
+ router = DocRouter.DEFAULT;
+ assertTrue(router instanceof CompositeIdRouter);
+
+ DocCollection coll = createCollection(4, router);
+ doNormalIdHashing(coll);
+
+ // ensure that the shard hashed to is only dependent on the first part of the compound key
+ doId(coll, "b!foo", "shard1");
+ doId(coll, "c!bar", "shard2");
+ doId(coll, "d!baz", "shard3");
+ doId(coll, "e!qux", "shard4");
+
+ // syntax to specify bits.
+ // Anything over 2 bits should give the same results as above (since only top 2 bits
+ // affect our 4 slice collection).
+ doId(coll, "b/2!foo", "shard1");
+ doId(coll, "c/2!bar", "shard2");
+ doId(coll, "d/2!baz", "shard3");
+ doId(coll, "e/2!qux", "shard4");
+
+ doId(coll, "b/32!foo", "shard1");
+ doId(coll, "c/32!bar", "shard2");
+ doId(coll, "d/32!baz", "shard3");
+ doId(coll, "e/32!qux", "shard4");
+
+ // no bits allocated to the first part (kind of odd why anyone would do that though)
+ doIndex(coll, "foo/0!b", "shard1");
+ doIndex(coll, "foo/0!c", "shard2");
+ doIndex(coll, "foo/0!d", "shard3");
+ doIndex(coll, "foo/0!e", "shard4");
+
+ // means cover whole range on the query side
+ doQuery(coll, "foo/0!", "shard1,shard2,shard3,shard4");
+
+ doQuery(coll, "b/1!", "shard1,shard2"); // top bit of hash(b)==1, so shard1 and shard2
+ doQuery(coll, "d/1!", "shard3,shard4"); // top bit of hash(b)==0, so shard3 and shard4
+ }
+
+ /***
+ public void testPrintHashCodes() throws Exception {
+ // from negative to positive, the upper bits of the hash ranges should be
+ // shard1: 11
+ // shard2: 10
+ // shard3: 00
+ // shard4: 01
+
+ String[] highBitsToShard = {"shard3","shard4","shard1","shard2"};
+
+
+ for (int i = 0; i<26; i++) {
+ String id = new String(Character.toChars('a'+i));
+ int hash = hash(id);
+ System.out.println("hash of " + id + " is " + Integer.toHexString(hash) + " high bits=" + (hash>>>30)
+ + " shard="+highBitsToShard[hash>>>30]);
+ }
+ }
+ ***/
+
+
+
+ DocCollection createCollection(int nSlices, DocRouter router) {
+ List<Range> ranges = router.partitionRange(nSlices, router.fullRange());
+
+ Map<String,Slice> slices = new HashMap<String,Slice>();
+ for (int i=0; i<ranges.size(); i++) {
+ Range range = ranges.get(i);
+ Slice slice = new Slice("shard"+(i+1), null, map("range",range));
+ slices.put(slice.getName(), slice);
+ }
+
+ DocCollection coll = new DocCollection("collection1", slices, null, router);
+ return coll;
+ }
+
+
+
+ // from negative to positive, the upper bits of the hash ranges should be
+ // shard1: top bits:10 80000000:bfffffff
+ // shard2: top bits:11 c0000000:ffffffff
+ // shard3: top bits:00 00000000:3fffffff
+ // shard4: top bits:01 40000000:7fffffff
+
+ /***
+ hash of a is 3c2569b2 high bits=0 shard=shard3
+ hash of b is 95de7e03 high bits=2 shard=shard1
+ hash of c is e132d65f high bits=3 shard=shard2
+ hash of d is 27191473 high bits=0 shard=shard3
+ hash of e is 656c4367 high bits=1 shard=shard4
+ hash of f is 2b64883b high bits=0 shard=shard3
+ hash of g is f18ae416 high bits=3 shard=shard2
+ hash of h is d482b2d3 high bits=3 shard=shard2
+ hash of i is 811a702b high bits=2 shard=shard1
+ hash of j is ca745a39 high bits=3 shard=shard2
+ hash of k is cfbda5d1 high bits=3 shard=shard2
+ hash of l is 1d5d6a2c high bits=0 shard=shard3
+ hash of m is 5ae4385c high bits=1 shard=shard4
+ hash of n is c651d8ac high bits=3 shard=shard2
+ hash of o is 68348473 high bits=1 shard=shard4
+ hash of p is 986fdf9a high bits=2 shard=shard1
+ hash of q is ff8209e8 high bits=3 shard=shard2
+ hash of r is 5c9373f1 high bits=1 shard=shard4
+ hash of s is ff4acaf1 high bits=3 shard=shard2
+ hash of t is ca87df4d high bits=3 shard=shard2
+ hash of u is 62203ae0 high bits=1 shard=shard4
+ hash of v is bdafcc55 high bits=2 shard=shard1
+ hash of w is ff439d1f high bits=3 shard=shard2
+ hash of x is 3e9a9b1b high bits=0 shard=shard3
+ hash of y is 477d9216 high bits=1 shard=shard4
+ hash of z is c1f69a17 high bits=3 shard=shard2
+ ***/
+
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Mon Dec 10 16:36:47 2012
@@ -40,7 +40,7 @@ public class ZkControllerTest extends So
private static final String COLLECTION_NAME = "collection1";
- static final int TIMEOUT = 1000;
+ static final int TIMEOUT = 10000;
private static final boolean DEBUG = false;
@@ -82,7 +82,7 @@ public class ZkControllerTest extends So
cc = getCoreContainer();
ZkController zkController = new ZkController(cc, server.getZkAddress(), TIMEOUT, 10000,
- "127.0.0.1", "8983", "solr", new CurrentCoreDescriptorProvider() {
+ "127.0.0.1", "8983", "solr", "0", new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {
@@ -122,7 +122,7 @@ public class ZkControllerTest extends So
cc = getCoreContainer();
zkController = new ZkController(cc, server.getZkAddress(),
- TIMEOUT, 10000, "127.0.0.1", "8983", "solr", new CurrentCoreDescriptorProvider() {
+ TIMEOUT, 10000, "127.0.0.1", "8983", "solr", "0", new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {
@@ -166,10 +166,5 @@ public class ZkControllerTest extends So
public void tearDown() throws Exception {
super.tearDown();
}
-
- @AfterClass
- public static void afterClass() throws InterruptedException {
- // wait just a bit for any zk client threads to outlast timeout
- Thread.sleep(2000);
- }
+
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java Mon Dec 10 16:36:47 2012
@@ -71,6 +71,32 @@ public class ZkSolrClientTest extends Ab
zkClient.close();
server.shutdown();
}
+
+ public void testClean() throws Exception {
+ String zkDir = dataDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+ ZkTestServer server = null;
+
+ server = new ZkTestServer(zkDir);
+ server.run();
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+
+
+ SolrZkClient zkClient = new SolrZkClient(server.getZkHost(),
+ AbstractZkTestCase.TIMEOUT);
+
+ zkClient.makePath("/test/path/here", true);
+
+ zkClient.makePath("/zz/path/here", true);
+
+ zkClient.clean("/");
+
+ assertFalse(zkClient.exists("/test", true));
+ assertFalse(zkClient.exists("/zz", true));
+
+ zkClient.close();
+ server.shutdown();
+ }
public void testReconnect() throws Exception {
String zkDir = dataDir.getAbsolutePath() + File.separator
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java Mon Dec 10 16:36:47 2012
@@ -19,9 +19,9 @@ package org.apache.solr.core;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.SimpleFloatFunction;
-import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.search.FunctionQParser;
+import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.ValueSourceParser;
@@ -38,7 +38,7 @@ public class DummyValueSourceParser exte
}
@Override
- public ValueSource parse(FunctionQParser fp) throws ParseException {
+ public ValueSource parse(FunctionQParser fp) throws SyntaxError {
ValueSource source = fp.parseValueSource();
ValueSource result = new SimpleFloatFunction(source) {
@Override
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java Mon Dec 10 16:36:47 2012
@@ -29,7 +29,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import org.apache.solr.common.SolrException;
@@ -86,7 +85,7 @@ public class TestArbitraryIndexDir exten
}
@Test
- public void testLoadNewIndexDir() throws IOException, ParserConfigurationException, SAXException, ParseException {
+ public void testLoadNewIndexDir() throws IOException, ParserConfigurationException, SAXException {
//add a doc in original index dir
assertU(adoc("id", String.valueOf(1),
"name", "name"+String.valueOf(1)));
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestConfig.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestConfig.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestConfig.java Mon Dec 10 16:36:47 2012
@@ -94,7 +94,7 @@ public class TestConfig extends SolrTest
@Test
public void testLucene23Upgrades() throws Exception {
double bufferSize = solrConfig.indexConfig.ramBufferSizeMB;
- assertTrue(bufferSize + " does not equal: " + 32, bufferSize == 32);
+ assertTrue(bufferSize + " does not equal: " + 100, bufferSize == 100);
String mergePolicy = solrConfig.indexConfig.mergePolicyInfo.className;
assertEquals(TieredMergePolicy.class.getName(), mergePolicy);
String mergeSched = solrConfig.indexConfig.mergeSchedulerInfo.className;
@@ -140,7 +140,7 @@ public class TestConfig extends SolrTest
public void testDefaults() throws Exception {
SolrConfig sc = new SolrConfig(new SolrResourceLoader("solr/collection1"), "solrconfig-basic.xml", null);
SolrIndexConfig sic = sc.indexConfig;
- assertTrue("default ramBufferSizeMB should be 32", sic.ramBufferSizeMB == 32);
+ assertTrue("default ramBufferSizeMB should be 100", sic.ramBufferSizeMB == 100);
assertTrue("default useCompoundFile should be false", sic.useCompoundFile == false);
assertTrue("default LockType should be native", sic.lockType.equals(SolrIndexConfig.LOCK_TYPE_NATIVE));
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestLazyCores.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestLazyCores.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/core/TestLazyCores.java Mon Dec 10 16:36:47 2012
@@ -17,6 +17,8 @@ package org.apache.solr.core;
* limitations under the License.
*/
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.NamedList;
@@ -27,18 +29,13 @@ import org.apache.solr.update.AddUpdateC
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.UpdateHandler;
import org.apache.solr.util.RefCounted;
-import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
-import org.xml.sax.SAXException;
-import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
-import java.util.Map;
public class TestLazyCores extends SolrTestCaseJ4 {
@@ -47,48 +44,77 @@ public class TestLazyCores extends SolrT
initCore("solrconfig.xml", "schema.xml");
}
- private CoreContainer cores;
+ private final File _solrHomeDirectory = new File(TEMP_DIR, "org.apache.solr.core.TestLazyCores_testlazy");
- @Before
- public void before() throws IOException, SAXException, ParserConfigurationException {
- String solrHome;
- solrHome = SolrResourceLoader.locateSolrHome();
- File fconf = new File(solrHome, "solr-lots-of-cores.xml");
+ private static String[] _necessaryConfs = {"schema.xml", "solrconfig.xml", "stopwords.txt", "synonyms.txt",
+ "protwords.txt", "old_synonyms.txt", "currency.xml", "open-exchange-rates.json", "mapping-ISOLatin1Accent.txt"};
+
+ private void copyConfFiles(File home, String subdir) throws IOException {
+
+ File subHome = new File(new File(home, subdir), "conf");
+ assertTrue("Failed to make subdirectory ", subHome.mkdirs());
+ String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf";
+ for (String file : _necessaryConfs) {
+ FileUtils.copyFile(new File(top, file), new File(subHome, file));
+ }
+ }
+
+ private CoreContainer init() throws Exception {
+
+ if (_solrHomeDirectory.exists()) {
+ FileUtils.deleteDirectory(_solrHomeDirectory);
+ }
+ assertTrue("Failed to mkdirs workDir", _solrHomeDirectory.mkdirs());
+ for (int idx = 1; idx < 10; ++idx) {
+ copyConfFiles(_solrHomeDirectory, "collection" + idx);
+ }
+
+ File solrXml = new File(_solrHomeDirectory, "solr.xml");
+ FileUtils.write(solrXml, LOTS_SOLR_XML, IOUtils.CHARSET_UTF_8.toString());
+ final CoreContainer cores = new CoreContainer(_solrHomeDirectory.getAbsolutePath());
+ cores.load(_solrHomeDirectory.getAbsolutePath(), solrXml);
+ // h.getCoreContainer().load(_solrHomeDirectory.getAbsolutePath(), new File(_solrHomeDirectory, "solr.xml"));
- cores = new CoreContainer(solrHome);
- cores.load(solrHome, fconf);
cores.setPersistent(false);
+ return cores;
}
+
+ public void after() throws Exception {
+ if (_solrHomeDirectory.exists()) {
+ FileUtils.deleteDirectory(_solrHomeDirectory);
+ }
+ }
+
@Test
- public void testLazyLoad() {
+ public void testLazyLoad() throws Exception {
+ CoreContainer cc = init();
try {
// NOTE: the way this works, this should not assert, however if it's put after the getCore on this collection,
// that will cause the core to be loaded and this test will fail.
- Collection<String> names = cores.getCoreNames();
+ Collection<String> names = cc.getCoreNames();
for (String name : names) {
assertFalse("collectionLazy2".equals(name));
}
- SolrCore core1 = cores.getCore("collection1");
- CoreDescriptor cont = core1.getCoreDescriptor();
+ SolrCore core1 = cc.getCore("collection1");
assertFalse("core1 should not be swappable", core1.getCoreDescriptor().isSwappable());
assertTrue("core1 should be loadable", core1.getCoreDescriptor().isLoadOnStartup());
assertNotNull(core1.getSolrConfig());
- SolrCore core2 = cores.getCore("collectionLazy2");
+ SolrCore core2 = cc.getCore("collectionLazy2");
assertTrue("core2 should not be swappable", core2.getCoreDescriptor().isSwappable());
assertFalse("core2 should not be loadable", core2.getCoreDescriptor().isLoadOnStartup());
- SolrCore core3 = cores.getCore("collectionLazy3");
+ SolrCore core3 = cc.getCore("collectionLazy3");
assertTrue("core3 should not be swappable", core3.getCoreDescriptor().isSwappable());
assertFalse("core3 should not be loadable", core3.getCoreDescriptor().isLoadOnStartup());
- SolrCore core4 = cores.getCore("collectionLazy4");
+ SolrCore core4 = cc.getCore("collectionLazy4");
assertFalse("core4 should not be swappable", core4.getCoreDescriptor().isSwappable());
assertFalse("core4 should not be loadable", core4.getCoreDescriptor().isLoadOnStartup());
- SolrCore core5 = cores.getCore("collectionLazy5");
+ SolrCore core5 = cc.getCore("collectionLazy5");
assertFalse("core5 should not be swappable", core5.getCoreDescriptor().isSwappable());
assertTrue("core5 should be loadable", core5.getCoreDescriptor().isLoadOnStartup());
@@ -98,7 +124,7 @@ public class TestLazyCores extends SolrT
core4.close();
core5.close();
} finally {
- cores.shutdown();
+ cc.shutdown();
}
}
@@ -106,10 +132,11 @@ public class TestLazyCores extends SolrT
// will, of course, load it.
@Test
public void testLazySearch() throws Exception {
+ CoreContainer cc = init();
try {
// Make sure Lazy2 isn't loaded.
- checkNotInCores("collectionLazy2");
- SolrCore core2 = cores.getCore("collectionLazy2");
+ checkNotInCores(cc, "collectionLazy2");
+ SolrCore core2 = cc.getCore("collectionLazy2");
addLazy(core2, "id", "0");
addLazy(core2, "id", "1", "v_t", "Hello Dude");
@@ -154,47 +181,45 @@ public class TestLazyCores extends SolrT
, "//result[@numFound='0']"
);
- checkInCores("collectionLazy2");
+ checkInCores(cc, "collectionLazy2");
searcher.close();
core2.close();
} finally {
- cores.shutdown();
+ cc.shutdown();
}
}
+
@Test
- public void testCachingLimit() {
+ public void testCachingLimit() throws Exception {
+ CoreContainer cc = init();
try {
- // NOTE: the way this works, this should not assert, however if it's put after the getCore on this collection,
- // that will cause the core to be loaded and this test will fail.
- Collection<String> names = cores.getCoreNames();
-
// By putting these in non-alpha order, we're also checking that we're not just seeing an artifact.
- SolrCore core1 = cores.getCore("collection1");
- SolrCore core2 = cores.getCore("collectionLazy3");
- SolrCore core4 = cores.getCore("collectionLazy4");
- SolrCore core3 = cores.getCore("collectionLazy2");
- SolrCore core5 = cores.getCore("collectionLazy5");
+ SolrCore core1 = cc.getCore("collection1");
+ SolrCore core2 = cc.getCore("collectionLazy3");
+ SolrCore core4 = cc.getCore("collectionLazy4");
+ SolrCore core3 = cc.getCore("collectionLazy2");
+ SolrCore core5 = cc.getCore("collectionLazy5");
- checkInCores("collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5");
- checkNotInCores("collectionLazy6", "collectionLazy7", "collectionLazy8", "collectionLazy9");
+ checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5");
+ checkNotInCores(cc, "collectionLazy6", "collectionLazy7", "collectionLazy8", "collectionLazy9");
// map should be full up, add one more and verify
- SolrCore core6 = cores.getCore("collectionLazy6");
- checkInCores("collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5", "collectionLazy6");
- checkNotInCores("collectionLazy7", "collectionLazy8", "collectionLazy9");
-
- SolrCore core7 = cores.getCore("collectionLazy7");
- checkInCores("collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5", "collectionLazy6", "collectionLazy7");
- checkNotInCores("collectionLazy8", "collectionLazy9");
- SolrCore core8 = cores.getCore("collectionLazy8");
- checkInCores("collection1", "collectionLazy2", "collectionLazy4", "collectionLazy5", "collectionLazy6", "collectionLazy7", "collectionLazy8");
- checkNotInCores("collectionLazy3", "collectionLazy9");
-
- SolrCore core9 = cores.getCore("collectionLazy9");
- checkInCores("collection1", "collectionLazy4", "collectionLazy5", "collectionLazy6", "collectionLazy7", "collectionLazy8", "collectionLazy9");
- checkNotInCores( "collectionLazy2","collectionLazy3");
+ SolrCore core6 = cc.getCore("collectionLazy6");
+ checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5", "collectionLazy6");
+ checkNotInCores(cc, "collectionLazy7", "collectionLazy8", "collectionLazy9");
+
+ SolrCore core7 = cc.getCore("collectionLazy7");
+ checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5", "collectionLazy6", "collectionLazy7");
+ checkNotInCores(cc, "collectionLazy8", "collectionLazy9");
+ SolrCore core8 = cc.getCore("collectionLazy8");
+ checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy4", "collectionLazy5", "collectionLazy6", "collectionLazy7", "collectionLazy8");
+ checkNotInCores(cc, "collectionLazy3", "collectionLazy9");
+
+ SolrCore core9 = cc.getCore("collectionLazy9");
+ checkInCores(cc, "collection1", "collectionLazy4", "collectionLazy5", "collectionLazy6", "collectionLazy7", "collectionLazy8", "collectionLazy9");
+ checkNotInCores(cc, "collectionLazy2", "collectionLazy3");
// Note decrementing the count when the core is removed from the lazyCores list is appropriate, since the
@@ -209,19 +234,19 @@ public class TestLazyCores extends SolrT
core8.close();
core9.close();
} finally {
- cores.shutdown();
+ cc.shutdown();
}
}
- private void checkNotInCores(String... nameCheck) {
- Collection<String> names = cores.getCoreNames();
+ private void checkNotInCores(CoreContainer cc, String... nameCheck) {
+ Collection<String> names = cc.getCoreNames();
for (String name : nameCheck) {
assertFalse("core " + name + " was found in the list of cores", names.contains(name));
}
}
- private void checkInCores(String... nameCheck) {
- Collection<String> names = cores.getCoreNames();
+ private void checkInCores(CoreContainer cc, String... nameCheck) {
+ Collection<String> names = cc.getCoreNames();
for (String name : nameCheck) {
assertTrue("core " + name + " was not found in the list of cores", names.contains(name));
}
@@ -251,10 +276,24 @@ public class TestLazyCores extends SolrT
if (q.length % 2 != 0) {
throw new RuntimeException("The length of the string array (query arguments) needs to be even");
}
- Map.Entry<String, String>[] entries = new NamedList.NamedListEntry[q.length / 2];
+ NamedList.NamedListEntry[] entries = new NamedList.NamedListEntry[q.length / 2];
for (int i = 0; i < q.length; i += 2) {
entries[i / 2] = new NamedList.NamedListEntry<String>(q[i], q[i + 1]);
}
- return new LocalSolrQueryRequest(core, new NamedList(entries));
+ return new LocalSolrQueryRequest(core, new NamedList<Object>(entries));
}
+
+ private final static String LOTS_SOLR_XML = " <solr persistent=\"false\"> " +
+ "<cores adminPath=\"/admin/cores\" defaultCoreName=\"collectionLazy2\" swappableCacheSize=\"4\"> " +
+ "<core name=\"collection1\" instanceDir=\"collection1\" /> " +
+ "<core name=\"collectionLazy2\" instanceDir=\"collection2\" swappable=\"true\" loadOnStartup=\"false\" /> " +
+ "<core name=\"collectionLazy3\" instanceDir=\"collection3\" swappable=\"on\" loadOnStartup=\"false\"/> " +
+ "<core name=\"collectionLazy4\" instanceDir=\"collection4\" swappable=\"false\" loadOnStartup=\"false\"/> " +
+ "<core name=\"collectionLazy5\" instanceDir=\"collection5\" swappable=\"false\" loadOnStartup=\"true\"/> " +
+ "<core name=\"collectionLazy6\" instanceDir=\"collection6\" swappable=\"true\" loadOnStartup=\"false\" /> " +
+ "<core name=\"collectionLazy7\" instanceDir=\"collection7\" swappable=\"true\" loadOnStartup=\"false\" /> " +
+ "<core name=\"collectionLazy8\" instanceDir=\"collection8\" swappable=\"true\" loadOnStartup=\"false\" /> " +
+ "<core name=\"collectionLazy9\" instanceDir=\"collection9\" swappable=\"true\" loadOnStartup=\"false\" /> " +
+ "</cores> " +
+ "</solr>";
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Mon Dec 10 16:36:47 2012
@@ -656,6 +656,8 @@ public class TestReplicationHandler exte
}
private void doTestReplicateAfterCoreReload() throws Exception {
+ int docs = TEST_NIGHTLY ? 200000 : 0;
+
//stop slave
slaveJetty.stop();
@@ -670,14 +672,14 @@ public class TestReplicationHandler exte
masterClient = createNewSolrServer(masterJetty.getLocalPort());
masterClient.deleteByQuery("*:*");
- for (int i = 0; i < 10; i++)
+ for (int i = 0; i < docs; i++)
index(masterClient, "id", i, "name", "name = " + i);
masterClient.commit();
- NamedList masterQueryRsp = rQuery(10, "*:*", masterClient);
+ NamedList masterQueryRsp = rQuery(docs, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
- assertEquals(10, masterQueryResult.getNumFound());
+ assertEquals(docs, masterQueryResult.getNumFound());
slave.setTestPort(masterJetty.getLocalPort());
slave.copyConfigFile(slave.getSolrConfigFile(), "solrconfig.xml");
@@ -687,9 +689,9 @@ public class TestReplicationHandler exte
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
//get docs from slave and check if number is equal to master
- NamedList slaveQueryRsp = rQuery(10, "*:*", slaveClient);
+ NamedList slaveQueryRsp = rQuery(docs, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
- assertEquals(10, slaveQueryResult.getNumFound());
+ assertEquals(docs, slaveQueryResult.getNumFound());
//compare results
String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
@@ -703,19 +705,19 @@ public class TestReplicationHandler exte
assertEquals(version, getIndexVersion(masterClient).get("indexversion"));
assertEquals(commits.get("commits"), getCommits(masterClient).get("commits"));
- index(masterClient, "id", 110, "name", "name = 1");
- index(masterClient, "id", 120, "name", "name = 2");
+ index(masterClient, "id", docs + 10, "name", "name = 1");
+ index(masterClient, "id", docs + 20, "name", "name = 2");
masterClient.commit();
- NamedList resp = rQuery(12, "*:*", masterClient);
+ NamedList resp = rQuery(docs + 2, "*:*", masterClient);
masterQueryResult = (SolrDocumentList) resp.get("response");
- assertEquals(12, masterQueryResult.getNumFound());
+ assertEquals(docs + 2, masterQueryResult.getNumFound());
//get docs from slave and check if number is equal to master
- slaveQueryRsp = rQuery(12, "*:*", slaveClient);
+ slaveQueryRsp = rQuery(docs + 2, "*:*", slaveClient);
slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
- assertEquals(12, slaveQueryResult.getNumFound());
+ assertEquals(docs + 2, slaveQueryResult.getNumFound());
// NOTE: revert config on master.
master.copyConfigFile(CONF_DIR + "solrconfig-master.xml", "solrconfig.xml");
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/response/TestCSVResponseWriter.java Mon Dec 10 16:36:47 2012
@@ -39,6 +39,8 @@ public class TestCSVResponseWriter exten
assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));
assertU(adoc("id","3", "shouldbeunstored","foo"));
+ assertU(adoc("id","4", "amount_c", "1.50,EUR"));
+ assertU(adoc("id","5", "store", "12.434,-134.1"));
assertU(commit());
}
@@ -69,6 +71,13 @@ public class TestCSVResponseWriter exten
assertEquals("2,hi|there,nice:output\n"
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "f.v2_ss.csv.separator",":", "fl","id,v_ss,v2_ss")));
+ // test csv field for polyfield (currency) SOLR-3959
+ assertEquals("4,\"1.50\\,EUR\"\n"
+ , h.query(req("q","id:4", "wt","csv", "csv.header","false", "fl","id,amount_c")));
+
+ // test csv field for polyfield (latlon) SOLR-3959
+ assertEquals("5,\"12.434\\,-134.1\"\n"
+ , h.query(req("q","id:5", "wt","csv", "csv.header","false", "fl","id,store")) );
// test retrieving fields from index
String result = h.query(req("q","*:*", "wt","csv", "csv.header","true", "fl","*,score"));
for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) {
@@ -100,7 +109,7 @@ public class TestCSVResponseWriter exten
, h.query(req("q","id:[1 TO 2]", "wt","csv", "csv.header","false", "fl","id,v_ss,foo_s")));
// test SOLR-2970 not returning non-stored fields by default. Compare sorted list
- assertEquals(sortHeader("v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l\n")
+ assertEquals(sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l\n")
, sortHeader(h.query(req("q","id:3", "wt","csv", "csv.header","true", "fl","*", "rows","0"))));
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java Mon Dec 10 16:36:47 2012
@@ -17,7 +17,6 @@
package org.apache.solr.search;
-import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
@@ -42,7 +41,7 @@ class FooQParser extends QParser {
}
@Override
- public Query parse() throws ParseException {
+ public Query parse() throws SyntaxError {
return new TermQuery(new Term(localParams.get(QueryParsing.F), localParams.get(QueryParsing.V)));
}
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java Mon Dec 10 16:36:47 2012
@@ -772,7 +772,7 @@ public class TestExtendedDismaxParser ex
"defType", "edismax")
, "*[count(//doc)=1]");
- // Query string field 'cat_s' for special char / - causes ParseException without patch SOLR-3467
+ // Query string field 'cat_s' for special char / - causes SyntaxError without patch SOLR-3467
assertQ("Escaping string with reserved / character",
req("q", "foo/",
"qf", "cat_s",
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java?rev=1419570&r1=1419569&r2=1419570&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java Mon Dec 10 16:36:47 2012
@@ -186,25 +186,36 @@ public class TestFoldingMultitermQuery e
}
@Test
+ public void testFuzzy() throws Exception {
+ assertQ(req("q", "content:ZiLLx~1"),
+ "//result[@numFound='1']");
+ assertQ(req("q", "content_straight:ZiLLx~1"), // case preserving field shouldn't match
+ "//result[@numFound='0']");
+ assertQ(req("q", "content_folding:ZiLLx~1"), // case preserving field shouldn't match
+ "//result[@numFound='0']");
+ }
+
+ @Test
public void testRegex() throws Exception {
assertQ(req("q", "content:/Zill[a-z]/"),
- "//result[@numFound='1']");
+ "//result[@numFound='1']");
assertQ(req("q", "content:/Zill[A-Z]/"), // everything in the regex gets lowercased?
- "//result[@numFound='1']");
+ "//result[@numFound='1']");
assertQ(req("q", "content_keyword:/.*Zill[A-Z]/"),
- "//result[@numFound='1']");
+ "//result[@numFound='1']");
assertQ(req("q", "content_straight:/Zill[a-z]/"), // case preserving field shouldn't match
- "//result[@numFound='0']");
+ "//result[@numFound='0']");
assertQ(req("q", "content_folding:/Zill[a-z]/"), // case preserving field shouldn't match
- "//result[@numFound='0']");
+ "//result[@numFound='0']");
assertQ(req("q", "content_keyword:/Abcdefg1 Finger/"), // test spaces
- "//result[@numFound='1']");
+ "//result[@numFound='1']");
}
+
@Test
public void testGeneral() throws Exception {
assertQ(req("q", "content_stemming:fings*"), "//result[@numFound='0']"); // should not match (but would if fings* was stemmed to fing*