You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ry...@apache.org on 2012/03/07 00:17:23 UTC
svn commit: r1297785 [14/16] - in
/lucene/dev/branches/lucene3795_lsp_spatial_module: ./ dev-tools/eclipse/
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/maven/lucene/contrib/demo/
dev-tools/maven/lucene/contrib/highlighter/ dev-tools/maven/lucene...
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/util/plugin/AbstractPluginLoader.java Tue Mar 6 23:17:08 2012
@@ -139,7 +139,7 @@ public abstract class AbstractPluginLoad
String defaultStr = DOMUtil.getAttr(node,"default", null );
T plugin = create(loader, name, className, node );
- log.info("created " + ((name != null) ? name : "") + ": " + plugin.getClass().getName());
+ log.debug("created " + ((name != null) ? name : "") + ": " + plugin.getClass().getName());
// Either initialize now or wait till everything has been registered
if( preRegister ) {
@@ -209,7 +209,7 @@ public abstract class AbstractPluginLoad
String name = DOMUtil.getAttr(node, "name", requireName ? type : null);
String className = DOMUtil.getAttr(node, "class", type);
plugin = create(loader, name, className, node);
- log.info("created " + name + ": " + plugin.getClass().getName());
+ log.debug("created " + name + ": " + plugin.getClass().getName());
// Either initialize now or wait till everything has been registered
if (preRegister) {
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test-files/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test-files/solr/conf/schema.xml?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test-files/solr/conf/schema.xml (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test-files/solr/conf/schema.xml Tue Mar 6 23:17:08 2012
@@ -538,6 +538,11 @@
<field name="nullfirst" type="string" indexed="true" stored="true" sortMissingFirst="true" multiValued="false"/>
+
+ <field name="cat" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="price" type="float" indexed="true" stored="true"/>
+ <field name="inStock" type="boolean" indexed="true" stored="true" />
+
<field name="subword" type="subword" indexed="true" stored="true"/>
<field name="subword_offsets" type="subword" indexed="true" stored="true" termOffsets="true"/>
<field name="numericsubword" type="numericsubword" indexed="true" stored="true"/>
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java Tue Mar 6 23:17:08 2012
@@ -18,6 +18,7 @@ package org.apache.solr;
*/
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
/**
@@ -169,6 +170,9 @@ public class TestDistributedGrouping ext
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " desc", "group.sort", "score desc"); // SOLR-2955
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", "score desc, _docid_ asc, id asc");
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10);
+
+ // Can't validate the response, but can check if no errors occur.
+ simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc", CommonParams.TIME_ALLOWED, 1);
}
private void simpleQuery(Object... queryParams) throws SolrServerException {
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedSearch.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestDistributedSearch.java Tue Mar 6 23:17:08 2012
@@ -17,7 +17,21 @@
package org.apache.solr;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.ChaosMonkey;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.NamedList;
/**
* TODO? perhaps use:
@@ -274,11 +288,137 @@ public class TestDistributedSearch exten
query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS);
query("q", "id:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY);
+ // Check Info is added to for each shard
+ ModifiableSolrParams q = new ModifiableSolrParams();
+ q.set("q", "*:*");
+ q.set(ShardParams.SHARDS_INFO, true);
+ setDistributedParams(q);
+ QueryResponse rsp = queryServer(q);
+ NamedList<?> sinfo = (NamedList<?>) rsp.getResponse().get(ShardParams.SHARDS_INFO);
+ String shards = getShardsString();
+ int cnt = StringUtils.countMatches(shards, ",")+1;
+
+ assertNotNull("missing shard info", sinfo);
+ assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, cnt, sinfo.size());
+
+ // test shards.tolerant=true
+ for(int numDownServers = 0; numDownServers < jettys.size()-1; numDownServers++)
+ {
+ List<JettySolrRunner> upJettys = new ArrayList<JettySolrRunner>(jettys);
+ List<SolrServer> upClients = new ArrayList<SolrServer>(clients);
+ List<JettySolrRunner> downJettys = new ArrayList<JettySolrRunner>();
+ List<String> upShards = new ArrayList<String>(Arrays.asList(shardsArr));
+ for(int i=0; i<numDownServers; i++)
+ {
+ // shut down some of the jettys
+ int indexToRemove = r.nextInt(upJettys.size());
+ JettySolrRunner downJetty = upJettys.remove(indexToRemove);
+ upClients.remove(indexToRemove);
+ upShards.remove(indexToRemove);
+ ChaosMonkey.stop(downJetty);
+ downJettys.add(downJetty);
+ }
+
+ queryPartialResults(upShards, upClients, "q","*:*",ShardParams.SHARDS_INFO,"true",ShardParams.SHARDS_TOLERANT,"true");
+
+ // restart the jettys
+ for (JettySolrRunner downJetty : downJettys) {
+ downJetty.start();
+ }
+ }
+
+ // This index has the same number for every field
+
// TODO: This test currently fails because debug info is obtained only
// on shards with matches.
// query("q","matchesnothing","fl","*,score", "debugQuery", "true");
// Thread.sleep(10000000000L);
}
+
+ protected void queryPartialResults(final List<String> upShards, List<SolrServer> upClients, Object... q) throws Exception {
+
+ final ModifiableSolrParams params = new ModifiableSolrParams();
+
+ for (int i = 0; i < q.length; i += 2) {
+ params.add(q[i].toString(), q[i + 1].toString());
+ }
+ // TODO: look into why passing true causes fails
+ params.set("distrib", "false");
+ final QueryResponse controlRsp = controlClient.query(params);
+ validateControlData(controlRsp);
+
+ params.remove("distrib");
+ setDistributedParams(params);
+
+ QueryResponse rsp = queryRandomUpServer(params,upClients);
+
+ comparePartialResponses(rsp, controlRsp, upShards);
+
+ if (stress > 0) {
+ log.info("starting stress...");
+ Thread[] threads = new Thread[nThreads];
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ for (int j = 0; j < stress; j++) {
+ int which = r.nextInt(clients.size());
+ SolrServer client = clients.get(which);
+ try {
+ QueryResponse rsp = client.query(new ModifiableSolrParams(params));
+ if (verifyStress) {
+ comparePartialResponses(rsp, controlRsp, upShards);
+ }
+ } catch (SolrServerException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+ threads[i].start();
+ }
+
+ for (Thread thread : threads) {
+ thread.join();
+ }
+ }
+ }
+
+ protected QueryResponse queryRandomUpServer(ModifiableSolrParams params, List<SolrServer> upClients) throws SolrServerException {
+ // query a random "up" server
+ int which = r.nextInt(upClients.size());
+ SolrServer client = upClients.get(which);
+ QueryResponse rsp = client.query(params);
+ return rsp;
+ }
+ protected void comparePartialResponses(QueryResponse rsp, QueryResponse controlRsp, List<String> upShards)
+ {
+ NamedList<?> sinfo = (NamedList<?>) rsp.getResponse().get(ShardParams.SHARDS_INFO);
+
+ assertNotNull("missing shard info", sinfo);
+ assertEquals("should have an entry for each shard ["+sinfo+"] "+shards, shardsArr.length, sinfo.size());
+ // identify each one
+ for (Map.Entry<String,?> entry : sinfo) {
+ String shard = entry.getKey();
+ NamedList<?> info = (NamedList<?>) entry.getValue();
+ boolean found = false;
+ for(int i=0; i<shardsArr.length; i++) {
+ String s = shardsArr[i];
+ if (shard.contains(s)) {
+ found = true;
+ // make sure that it responded if it's up
+ if (upShards.contains(s)) {
+ assertTrue("Expected to find numFound in the up shard info",info.get("numFound") != null);
+ }
+ else {
+ assertTrue("Expected to find error in the down shard info",info.get("error") != null);
+ }
+ }
+ }
+ assertTrue("Couldn't find shard " + shard + " represented in shards info", found);
+ }
+ }
+
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Tue Mar 6 23:17:08 2012
@@ -215,6 +215,19 @@ public class TestGroupingSearch extends
}
@Test
+ public void testGroupingWithTimeAllowed() throws Exception {
+ assertU(add(doc("id", "1")));
+ assertU(add(doc("id", "2")));
+ assertU(add(doc("id", "3")));
+ assertU(add(doc("id", "4")));
+ assertU(add(doc("id", "5")));
+ assertU(commit());
+
+ // Just checking if no errors occur
+ assertJQ(req("q", "*:*", "group", "true", "group.query", "id:1", "group.query", "id:2", "timeAllowed", "1"));
+ }
+
+ @Test
public void testGroupingSortByFunction() throws Exception {
assertU(add(doc("id", "1", "value1_i", "1", "value2_i", "1", "store", "45.18014,-93.87742")));
assertU(add(doc("id", "2", "value1_i", "1", "value2_i", "2", "store", "45.18014,-93.87743")));
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java Tue Mar 6 23:17:08 2012
@@ -50,7 +50,7 @@ public class TestKuromojiTokenizerFactor
factory.inform(new SolrResourceLoader(null, null));
TokenStream ts = factory.create(new StringReader("ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢"));
assertTokenStreamContents(ts,
- new String[] { "ã·ãã¢", "ã½ããã¦ã§ã¢", "ã¨ã³ã¸ãã¢" }
+ new String[] { "ã·ãã¢", "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢", "ã½ããã¦ã§ã¢", "ã¨ã³ã¸ãã¢" }
);
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java Tue Mar 6 23:17:08 2012
@@ -21,6 +21,7 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.solr.SolrTestCaseJ4;
@@ -211,12 +212,12 @@ public class TestWordDelimiterFilterFact
TokenStream ts = factoryDefault.create(
new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "5", "400", "00", "540000", "at", "25", "interest", "rate", "interestrate" });
ts = factoryDefault.create(
new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo", "bar", "foobar" });
@@ -229,13 +230,13 @@ public class TestWordDelimiterFilterFact
ts = factoryCustom.create(
new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "$5,400.00", "at", "25%", "interest", "rate", "interestrate" });
/* test custom behavior with a char > 0x7F, because we had to make a larger byte[] */
ts = factoryCustom.create(
new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
- BaseTokenTestCase.assertTokenStreamContents(ts,
+ BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo\u200Dbar" });
}
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java Tue Mar 6 23:17:08 2012
@@ -27,9 +27,9 @@ import org.apache.solr.common.cloud.Slic
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.zookeeper.KeeperException;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Before;
public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearchTestCase {
@@ -80,6 +80,14 @@ public abstract class AbstractDistribute
}
shards = sb.toString();
+
+ // now wait till we see the leader for each shard
+ for (int i = 1; i <= numShards; i++) {
+ ZkStateReader zkStateReader = ((SolrDispatchFilter) jettys.get(0)
+ .getDispatchFilter().getFilter()).getCores().getZkController()
+ .getZkStateReader();
+ zkStateReader.getLeaderProps("collection1", "shard" + (i + 2), 15000);
+ }
}
protected void waitForRecoveriesToFinish(String collection, ZkStateReader zkStateReader, boolean verbose)
@@ -109,7 +117,7 @@ public abstract class AbstractDistribute
ZkStateReader.NODE_NAME_PROP)));
String state = shard.getValue().get(ZkStateReader.STATE_PROP);
if ((state.equals(ZkStateReader.RECOVERING) || state
- .equals(ZkStateReader.SYNC))
+ .equals(ZkStateReader.SYNC) || state.equals(ZkStateReader.DOWN))
&& cloudState.liveNodesContain(shard.getValue().get(
ZkStateReader.NODE_NAME_PROP))) {
sawLiveRecovering = true;
@@ -151,7 +159,7 @@ public abstract class AbstractDistribute
String state = shard.getValue().get(ZkStateReader.STATE_PROP);
if (!state.equals(ZkStateReader.ACTIVE)) {
- fail("Not all shards are ACTIVE");
+ fail("Not all shards are ACTIVE - found a shard that is: " + state);
}
}
}
@@ -180,8 +188,4 @@ public abstract class AbstractDistribute
zkClient.printLayoutToStdOut();
zkClient.close();
}
-
- @AfterClass
- public static void afterClass() throws InterruptedException {
- }
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java Tue Mar 6 23:17:08 2012
@@ -24,10 +24,8 @@ import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.core.SolrConfig;
import org.apache.zookeeper.CreateMode;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -61,7 +59,7 @@ public abstract class AbstractZkTestCase
System.setProperty("solrcloud.skip.autorecovery", "true");
System.setProperty("zkHost", zkServer.getZkAddress());
- System.setProperty("hostPort", "0000");
+ System.setProperty("jetty.port", "0000");
buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(),
"solrconfig.xml", "schema.xml");
@@ -120,6 +118,8 @@ public abstract class AbstractZkTestCase
System.clearProperty("solr.test.sys.prop1");
System.clearProperty("solr.test.sys.prop2");
System.clearProperty("solrcloud.skip.autorecovery");
+ System.clearProperty("jetty.port");
+
zkServer.shutdown();
// wait just a bit for any zk client threads to outlast timeout
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Tue Mar 6 23:17:08 2012
@@ -22,25 +22,41 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.update.SolrCmdDistributor.Request;
+import org.apache.solr.util.DefaultSolrThreadFactory;
/**
*
*/
+
public class BasicDistributedZkTest extends AbstractDistributedZkTestCase {
private static final String DEFAULT_COLLECTION = "collection1";
@@ -63,12 +79,23 @@ public class BasicDistributedZkTest exte
String invalidField="ignore_exception__invalid_field_not_in_schema";
private Map<String,List<SolrServer>> otherCollectionClients = new HashMap<String,List<SolrServer>>();
- private Map<String,List<SolrServer>> oneInstanceCollectionClients = new HashMap<String,List<SolrServer>>();
+
private String oneInstanceCollection = "oneInstanceCollection";
private String oneInstanceCollection2 = "oneInstanceCollection2";
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(0,
+ Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ new DefaultSolrThreadFactory("testExecutor"));
+
+ CompletionService<Request> completionService;
+ Set<Future<Request>> pending;
+
public BasicDistributedZkTest() {
fixShardCount = true;
+ shardCount = 3;
+ completionService = new ExecutorCompletionService<Request>(executor);
+ pending = new HashSet<Future<Request>>();
+
}
@Override
@@ -251,36 +278,80 @@ public class BasicDistributedZkTest exte
testANewCollectionInOneInstance();
testSearchByCollectionName();
testANewCollectionInOneInstanceWithManualShardAssignement();
+ testNumberOfCommitsWithCommitAfterAdd();
+
// Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
}
}
+ private void testNumberOfCommitsWithCommitAfterAdd()
+ throws MalformedURLException, SolrServerException, IOException {
+ long startCommits = getNumCommits((CommonsHttpSolrServer) clients.get(0));
+
+ ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update/csv");
+ up.addFile(getFile("books_numeric_ids.csv"));
+ up.setCommitWithin(900000);
+ up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+ NamedList<Object> result = clients.get(0).request(up);
+
+ long endCommits = getNumCommits((CommonsHttpSolrServer) clients.get(0));
+
+ assertEquals(startCommits + 1L, endCommits);
+ }
+
+ private Long getNumCommits(CommonsHttpSolrServer solrServer) throws MalformedURLException,
+ SolrServerException, IOException {
+ CommonsHttpSolrServer server = new CommonsHttpSolrServer(solrServer.getBaseURL());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("qt", "/admin/mbeans?key=updateHandler&stats=true");
+ // use generic request to avoid extra processing of queries
+ QueryRequest req = new QueryRequest(params);
+ NamedList<Object> resp = server.request(req);
+ NamedList mbeans = (NamedList) resp.get("solr-mbeans");
+ NamedList uhandlerCat = (NamedList) mbeans.get("UPDATEHANDLER");
+ NamedList uhandler = (NamedList) uhandlerCat.get("updateHandler");
+ NamedList stats = (NamedList) uhandler.get("stats");
+ Long commits = (Long) stats.get("commits");
+
+ return commits;
+ }
+
private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
- oneInstanceCollectionClients.put(oneInstanceCollection , collectionClients);
+ otherCollectionClients.put(oneInstanceCollection2, collectionClients);
String baseUrl = ((CommonsHttpSolrServer) client).getBaseURL();
createCollection(oneInstanceCollection2, collectionClients, baseUrl, 1, "slice1");
createCollection(oneInstanceCollection2, collectionClients, baseUrl, 2, "slice2");
createCollection(oneInstanceCollection2, collectionClients, baseUrl, 3, "slice2");
createCollection(oneInstanceCollection2, collectionClients, baseUrl, 4, "slice1");
+ while (pending != null && pending.size() > 0) {
+
+ Future<Request> future = completionService.take();
+ pending.remove(future);
+ }
+
SolrServer client1 = createNewSolrServer(oneInstanceCollection2 + "1", baseUrl);
SolrServer client2 = createNewSolrServer(oneInstanceCollection2 + "2", baseUrl);
SolrServer client3 = createNewSolrServer(oneInstanceCollection2 + "3", baseUrl);
SolrServer client4 = createNewSolrServer(oneInstanceCollection2 + "4", baseUrl);
- client2.add(getDoc(id, "1"));
- client3.add(getDoc(id, "2"));
- client4.add(getDoc(id, "3"));
-
+
// no one should be recovering
waitForRecoveriesToFinish(oneInstanceCollection2, solrj.getZkStateReader(), false, true);
assertAllActive(oneInstanceCollection2, solrj.getZkStateReader());
+ // TODO: enable when we don't falsly get slice1...
+ // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000);
+ // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000);
+ client2.add(getDoc(id, "1"));
+ client3.add(getDoc(id, "2"));
+ client4.add(getDoc(id, "3"));
+
client1.commit();
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
@@ -299,9 +370,9 @@ public class BasicDistributedZkTest exte
// System.out.println("4:" + fourDocs);
// System.out.println("All Docs:" + allDocs);
- assertEquals(oneDocs, threeDocs);
- assertEquals(twoDocs, fourDocs);
- assertNotSame(oneDocs, twoDocs);
+// assertEquals(oneDocs, threeDocs);
+// assertEquals(twoDocs, fourDocs);
+// assertNotSame(oneDocs, twoDocs);
assertEquals(3, allDocs);
// we added a role of none on these creates - check for it
@@ -309,7 +380,7 @@ public class BasicDistributedZkTest exte
zkStateReader.updateCloudState(true);
Map<String,Slice> slices = zkStateReader.getCloudState().getSlices(oneInstanceCollection2);
assertNotNull(slices);
- String roles = slices.get("shard1").getShards().values().iterator().next().get(ZkStateReader.ROLES_PROP);
+ String roles = slices.get("slice1").getShards().values().iterator().next().get(ZkStateReader.ROLES_PROP);
assertEquals("none", roles);
}
@@ -328,13 +399,20 @@ public class BasicDistributedZkTest exte
private void testANewCollectionInOneInstance() throws Exception {
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
- oneInstanceCollectionClients.put(oneInstanceCollection , collectionClients);
+ otherCollectionClients.put(oneInstanceCollection , collectionClients);
String baseUrl = ((CommonsHttpSolrServer) client).getBaseURL();
createCollection(oneInstanceCollection, collectionClients, baseUrl, 1);
createCollection(oneInstanceCollection, collectionClients, baseUrl, 2);
createCollection(oneInstanceCollection, collectionClients, baseUrl, 3);
createCollection(oneInstanceCollection, collectionClients, baseUrl, 4);
+ while (pending != null && pending.size() > 0) {
+
+ Future<Request> future = completionService.take();
+ if (future == null) return;
+ pending.remove(future);
+ }
+
SolrServer client1 = createNewSolrServer(oneInstanceCollection + "1", baseUrl);
SolrServer client2 = createNewSolrServer(oneInstanceCollection + "2", baseUrl);
SolrServer client3 = createNewSolrServer(oneInstanceCollection + "3", baseUrl);
@@ -365,32 +443,49 @@ public class BasicDistributedZkTest exte
// System.out.println("4:" + fourDocs);
// System.out.println("All Docs:" + allDocs);
- assertEquals(oneDocs, threeDocs);
- assertEquals(twoDocs, fourDocs);
- assertNotSame(oneDocs, twoDocs);
assertEquals(3, allDocs);
}
private void createCollection(String collection,
List<SolrServer> collectionClients, String baseUrl, int num)
- throws MalformedURLException, SolrServerException, IOException {
+ throws MalformedURLException, SolrServerException, IOException, InterruptedException {
createCollection(collection, collectionClients, baseUrl, num, null);
}
- private void createCollection(String collection,
- List<SolrServer> collectionClients, String baseUrl, int num, String shardId)
- throws MalformedURLException, SolrServerException, IOException {
- CommonsHttpSolrServer server = new CommonsHttpSolrServer(
- baseUrl);
- Create createCmd = new Create();
- createCmd.setRoles("none");
- createCmd.setCoreName(collection + num);
- createCmd.setCollection(collection);
- createCmd.setNumShards(2);
- createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
- + collection + num);
- createCmd.setShardId(shardId);
- server.request(createCmd);
+ private void createCollection(final String collection,
+ List<SolrServer> collectionClients, final String baseUrl, final int num,
+ final String shardId) throws MalformedURLException, SolrServerException,
+ IOException, InterruptedException {
+ Callable call = new Callable() {
+ public Object call() {
+ CommonsHttpSolrServer server;
+ try {
+ server = new CommonsHttpSolrServer(baseUrl);
+
+ Create createCmd = new Create();
+ createCmd.setRoles("none");
+ createCmd.setCoreName(collection + num);
+ createCmd.setCollection(collection);
+ if (shardId == null) {
+ createCmd.setNumShards(2);
+ }
+ createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ + collection + num);
+ if (shardId != null) {
+ createCmd.setShardId(shardId);
+ }
+ server.request(createCmd);
+ } catch (Exception e) {
+ e.printStackTrace();
+ //fail
+ }
+ return null;
+ }
+ };
+
+ pending.add(completionService.submit(call));
+
+
collectionClients.add(createNewSolrServer(collection, baseUrl));
}
@@ -398,11 +493,20 @@ public class BasicDistributedZkTest exte
SolrServerException, IOException, Exception {
// create another 2 collections and search across them
createNewCollection("collection2");
+ createNewCollection("collection3");
+
+ while (pending != null && pending.size() > 0) {
+
+ Future<Request> future = completionService.take();
+ if (future == null) return;
+ pending.remove(future);
+ }
+
indexDoc("collection2", getDoc(id, "10000000"));
indexDoc("collection2", getDoc(id, "10000001"));
indexDoc("collection2", getDoc(id, "10000003"));
- createNewCollection("collection3");
+
indexDoc("collection3", getDoc(id, "20000000"));
indexDoc("collection3", getDoc(id, "20000001"));
@@ -455,21 +559,45 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(which);
client.add(doc);
}
-
- private void createNewCollection(String collection)
- throws MalformedURLException, SolrServerException, IOException {
- List<SolrServer> collectionClients = new ArrayList<SolrServer>();
+
+ private void createNewCollection(final String collection)
+ throws MalformedURLException, SolrServerException, IOException, InterruptedException {
+ final List<SolrServer> collectionClients = new ArrayList<SolrServer>();
otherCollectionClients.put(collection, collectionClients);
int unique = 0;
- for (SolrServer client : clients) {
- CommonsHttpSolrServer server = new CommonsHttpSolrServer(
- ((CommonsHttpSolrServer) client).getBaseURL());
- Create createCmd = new Create();
- createCmd.setCoreName(collection);
- createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator + collection + unique++);
- server.request(createCmd);
+ for (final SolrServer client : clients) {
+ unique++;
+ final int frozeUnique = unique;
+ Callable call = new Callable() {
+ public Object call() {
+ CommonsHttpSolrServer server;
+ try {
+ server = new CommonsHttpSolrServer(
+ ((CommonsHttpSolrServer) client).getBaseURL());
+
+ Create createCmd = new Create();
+ createCmd.setCoreName(collection);
+ createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ + collection + frozeUnique);
+ server.request(createCmd);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ //fails
+ }
+ return null;
+ }
+ };
+
collectionClients.add(createNewSolrServer(collection,
((CommonsHttpSolrServer) client).getBaseURL()));
+ pending.add(completionService.submit(call));
+ while (pending != null && pending.size() > 0) {
+
+ Future<Request> future = completionService.take();
+ if (future == null) return;
+ pending.remove(future);
+ }
}
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java Tue Mar 6 23:17:08 2012
@@ -46,9 +46,13 @@ public class BasicZkTest extends Abstrac
@Test
public void testBasic() throws Exception {
+
// test using ZooKeeper
assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware());
+ // for the really slow/busy computer, we wait to make sure we have a leader before starting
+ h.getCoreContainer().getZkController().getZkStateReader().getLeaderUrl("collection1", "shard1", 30000);
+
ZkController zkController = h.getCoreContainer().getZkController();
// test merge factor picked up
@@ -154,6 +158,7 @@ public class BasicZkTest extends Abstrac
}
+ zkController.getZkClient().printLayoutToStdOut();
}
public SolrQueryRequest request(String... q) {
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Tue Mar 6 23:17:08 2012
@@ -65,6 +65,7 @@ public class ChaosMonkeyNothingIsSafeTes
@Override
@After
public void tearDown() throws Exception {
+ System.clearProperty("numShards");
super.tearDown();
resetExceptionIgnores();
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Tue Mar 6 23:17:08 2012
@@ -32,7 +32,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
-@Ignore("Fixme! I am so tired of failing all the time. This is cruelty to animals! :(")
+@Ignore("SOLR-3126")
public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest {
@BeforeClass
@@ -61,6 +61,7 @@ public class ChaosMonkeySafeLeaderTest e
@Override
@After
public void tearDown() throws Exception {
+ System.clearProperty("numShards");
super.tearDown();
resetExceptionIgnores();
}
@@ -113,7 +114,7 @@ public class ChaosMonkeySafeLeaderTest e
waitForThingsToLevelOut();
- checkShardConsistency(true, false);
+ checkShardConsistency(true, true);
if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Tue Mar 6 23:17:08 2012
@@ -21,11 +21,7 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -37,6 +33,8 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.CloudState;
@@ -62,7 +60,12 @@ import org.junit.Ignore;
*/
@Ignore
public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
-
+ @BeforeClass
+ public static void beforeFullSolrCloudTest() throws Exception {
+ // shorten the log output more for this test type
+ if (formatter != null) formatter.setShorterFormat();
+ }
+
private static final String SHARD2 = "shard2";
private boolean printLayoutOnTearDown = false;
@@ -140,7 +143,7 @@ public class FullSolrCloudTest extends A
@Override
public void setUp() throws Exception {
super.setUp();
- ignoreException(".*");
+ // ignoreException(".*");
System.setProperty("numShards", Integer.toString(sliceCount));
}
@@ -638,7 +641,7 @@ public class FullSolrCloudTest extends A
// new server should be part of first shard
// how many docs are on the new shard?
for (SolrServer client : shardToClient.get("shard1")) {
- if (VERBOSE) System.out.println("total:"
+ if (VERBOSE) System.err.println("total:"
+ client.query(new SolrQuery("*:*")).getResults().getNumFound());
}
@@ -660,7 +663,7 @@ public class FullSolrCloudTest extends A
commit();
long deadShardCount = shardToClient.get(SHARD2).get(0).query(query).getResults().getNumFound();
- System.out.println("dsc:" + deadShardCount);
+ System.err.println("dsc:" + deadShardCount);
query("q", "*:*", "sort", "n_tl1 desc");
@@ -745,14 +748,14 @@ public class FullSolrCloudTest extends A
testDebugQueries();
if (VERBOSE) {
- System.out.println(controlClient.query(new SolrQuery("*:*")).getResults()
+ System.err.println(controlClient.query(new SolrQuery("*:*")).getResults()
.getNumFound());
for (SolrServer client : clients) {
try {
SolrQuery q = new SolrQuery("*:*");
q.set("distrib", false);
- System.out.println(client.query(q).getResults()
+ System.err.println(client.query(q).getResults()
.getNumFound());
} catch (Exception e) {
@@ -989,7 +992,7 @@ public class FullSolrCloudTest extends A
long num = -1;
long lastNum = -1;
String failMessage = null;
- if (verbose) System.out.println("check const of " + shard);
+ if (verbose) System.err.println("check const of " + shard);
int cnt = 0;
assertEquals(
@@ -998,17 +1001,18 @@ public class FullSolrCloudTest extends A
zkStateReader.getCloudState().getSlice(DEFAULT_COLLECTION, shard)
.getShards().size(), solrClients.size());
+ SolrServer lastClient = null;
for (SolrServer client : solrClients) {
ZkNodeProps props = clientToInfo.get(new CloudSolrServerClient(client));
- if (verbose) System.out.println("client" + cnt++);
- if (verbose) System.out.println("PROPS:" + props);
+ if (verbose) System.err.println("client" + cnt++);
+ if (verbose) System.err.println("PROPS:" + props);
try {
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
num = client.query(query).getResults().getNumFound();
} catch (SolrServerException e) {
- if (verbose) System.out.println("error contacting client: "
+ if (verbose) System.err.println("error contacting client: "
+ e.getMessage() + "\n");
continue;
}
@@ -1018,25 +1022,68 @@ public class FullSolrCloudTest extends A
if (zkStateReader.getCloudState().liveNodesContain(nodeName)) {
live = true;
}
- if (verbose) System.out.println(" live:" + live);
+ if (verbose) System.err.println(" live:" + live);
- if (verbose) System.out.println(" num:" + num + "\n");
+ if (verbose) System.err.println(" num:" + num + "\n");
boolean active = props.get(ZkStateReader.STATE_PROP).equals(
ZkStateReader.ACTIVE);
if (active && live) {
if (lastNum > -1 && lastNum != num && failMessage == null) {
- failMessage = shard + " is not consistent, expected:" + lastNum
- + " and got:" + num;
+ failMessage = shard + " is not consistent. Got " + lastNum + " from " + lastClient + "lastClient"
+ + " and got " + num + " from " + client;
+
+ if (verbose || true) {
+ System.err.println("######" + failMessage);
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+ query.set("fl","id,_version_");
+ query.set("rows","1000");
+ query.set("sort","id asc");
+
+ SolrDocumentList lst1 = lastClient.query(query).getResults();
+ SolrDocumentList lst2 = client.query(query).getResults();
+
+ showDiff(lst1, lst2, lastClient.toString(), client.toString());
+ }
+
}
lastNum = num;
+ lastClient = client;
}
}
-
return failMessage;
}
+ void showDiff(SolrDocumentList a, SolrDocumentList b, String aName, String bName) {
+ System.err.println("######"+aName+ ": " + a);
+ System.err.println("######"+bName+ ": " + b);
+ System.err.println("###### sizes=" + a.size() + "," + b.size());
+
+ Set<Map> setA = new HashSet<Map>();
+ for (SolrDocument sdoc : a) {
+ setA.add(new HashMap(sdoc));
+ }
+
+ Set<Map> setB = new HashSet<Map>();
+ for (SolrDocument sdoc : b) {
+ setB.add(new HashMap(sdoc));
+ }
+
+ Set<Map> onlyInA = new HashSet<Map>(setA);
+ onlyInA.removeAll(setB);
+ Set<Map> onlyInB = new HashSet<Map>(setB);
+ onlyInB.removeAll(setA);
+
+ if (onlyInA.size() > 0) {
+ System.err.println("###### Only in " + aName + ": " + onlyInA);
+ }
+ if (onlyInB.size() > 0) {
+ System.err.println("###### Only in " + bName + ": " + onlyInB);
+ }
+ }
+
protected void checkShardConsistency() throws Exception {
checkShardConsistency(true, false);
}
@@ -1045,7 +1092,7 @@ public class FullSolrCloudTest extends A
throws Exception {
long docs = controlClient.query(new SolrQuery("*:*")).getResults()
.getNumFound();
- if (verbose) System.out.println("Control Docs:" + docs);
+ if (verbose) System.err.println("Control Docs:" + docs);
updateMappingsFromZk(jettys, clients);
@@ -1079,9 +1126,9 @@ public class FullSolrCloudTest extends A
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
long results = client.query(query).getResults().getNumFound();
- if (verbose) System.out.println(new ZkCoreNodeProps(props)
+ if (verbose) System.err.println(new ZkCoreNodeProps(props)
.getCoreUrl() + " : " + results);
- if (verbose) System.out.println("shard:"
+ if (verbose) System.err.println("shard:"
+ props.get(ZkStateReader.SHARD_ID_PROP));
cnt += results;
break;
@@ -1116,7 +1163,7 @@ public class FullSolrCloudTest extends A
// TODO: as we create the clients, we should build a map from shard to
// node/client
// and node/client to shard?
- if (verbose) System.out.println("control docs:"
+ if (verbose) System.err.println("control docs:"
+ controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()
+ "\n\n");
long controlCount = controlClient.query(new SolrQuery("*:*")).getResults()
@@ -1148,8 +1195,8 @@ public class FullSolrCloudTest extends A
((CommonsHttpSolrServer) client).getBaseURL()).getPort()
+ "_solr_";
if (verbose && shard.getKey().endsWith(shardName)) {
- System.out.println("shard:" + slice.getKey());
- System.out.println(shard.getValue());
+ System.err.println("shard:" + slice.getKey());
+ System.err.println(shard.getValue());
}
}
}
@@ -1163,9 +1210,9 @@ public class FullSolrCloudTest extends A
count = client.query(query).getResults().getNumFound();
}
- if (verbose) System.out.println("client docs:" + count + "\n\n");
+ if (verbose) System.err.println("client docs:" + count + "\n\n");
}
- if (verbose) System.out.println("control docs:"
+ if (verbose) System.err.println("control docs:"
+ controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()
+ "\n\n");
SolrQuery query = new SolrQuery("*:*");
@@ -1265,6 +1312,7 @@ public class FullSolrCloudTest extends A
super.tearDown();
System.clearProperty("zkHost");
+ System.clearProperty("numShards");
}
protected void commit() throws Exception {
@@ -1292,7 +1340,7 @@ public class FullSolrCloudTest extends A
+ DEFAULT_COLLECTION;
CommonsHttpSolrServer s = new CommonsHttpSolrServer(url);
s.setConnectionTimeout(100); // 1/10th sec
- s.setSoTimeout(30000);
+ s.setSoTimeout(15000);
s.setDefaultMaxConnectionsPerHost(100);
s.setMaxTotalConnections(100);
return s;
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java Tue Mar 6 23:17:08 2012
@@ -74,6 +74,9 @@ public class LeaderElectionIntegrationTe
public void setUp() throws Exception {
super.setUp();
createTempDir();
+ ignoreException("No UpdateLog found - cannot sync");
+ ignoreException("No UpdateLog found - cannot recover");
+
System.setProperty("zkClientTimeout", "3000");
zkDir = dataDir.getAbsolutePath() + File.separator
@@ -268,6 +271,12 @@ public class LeaderElectionIntegrationTe
@AfterClass
public static void afterClass() throws InterruptedException {
System.clearProperty("solrcloud.skip.autorecovery");
+ System.clearProperty("zkClientTimeout");
+ System.clearProperty("zkHost");
+ System.clearProperty("shard");
+ System.clearProperty("solr.data.dir");
+ System.clearProperty("solr.solr.home");
+ resetExceptionIgnores();
// wait just a bit for any zk client threads to outlast timeout
Thread.sleep(2000);
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java Tue Mar 6 23:17:08 2012
@@ -110,7 +110,7 @@ public class LeaderElectionTest extends
try {
elector.setup(context);
- seq = elector.joinElection(context, null);
+ seq = elector.joinElection(context);
electionDone = true;
seqToThread.put(seq, this);
} catch (InterruptedException e) {
@@ -153,15 +153,42 @@ public class LeaderElectionTest extends
ElectionContext context = new ShardLeaderElectionContextBase(elector,
"shard2", "collection1", "dummynode1", props, zkStateReader);
elector.setup(context);
- elector.joinElection(context, null);
+ elector.joinElection(context);
assertEquals("http://127.0.0.1/solr/",
getLeaderUrl("collection1", "shard2"));
}
-
+
+ @Test
+ public void testCancelElection() throws Exception {
+ LeaderElector first = new LeaderElector(zkClient);
+ ZkNodeProps props = new ZkNodeProps(ZkStateReader.BASE_URL_PROP,
+ "http://127.0.0.1/solr/", ZkStateReader.CORE_NAME_PROP, "1");
+ ElectionContext firstContext = new ShardLeaderElectionContextBase(first,
+ "slice1", "collection2", "dummynode1", props, zkStateReader);
+ first.setup(firstContext);
+ first.joinElection(firstContext);
+
+ Thread.sleep(1000);
+ assertEquals("original leader was not registered", "http://127.0.0.1/solr/1/", getLeaderUrl("collection2", "slice1"));
+
+ LeaderElector second = new LeaderElector(zkClient);
+ props = new ZkNodeProps(ZkStateReader.BASE_URL_PROP,
+ "http://127.0.0.1/solr/", ZkStateReader.CORE_NAME_PROP, "2");
+ ElectionContext context = new ShardLeaderElectionContextBase(second,
+ "slice1", "collection2", "dummynode1", props, zkStateReader);
+ second.setup(context);
+ second.joinElection(context);
+ Thread.sleep(1000);
+ assertEquals("original leader should have stayed leader", "http://127.0.0.1/solr/1/", getLeaderUrl("collection2", "slice1"));
+ firstContext.cancelElection();
+ Thread.sleep(1000);
+ assertEquals("new leader was not registered", "http://127.0.0.1/solr/2/", getLeaderUrl("collection2", "slice1"));
+ }
+
private String getLeaderUrl(final String collection, final String slice)
throws KeeperException, InterruptedException {
int iterCount = 60;
- while (iterCount-- > 0)
+ while (iterCount-- > 0) {
try {
byte[] data = zkClient.getData(
ZkStateReader.getShardLeadersPath(collection, slice), null, null,
@@ -172,6 +199,8 @@ public class LeaderElectionTest extends
} catch (NoNodeException e) {
Thread.sleep(500);
}
+ }
+ zkClient.printLayoutToStdOut();
throw new RuntimeException("Could not get leader props");
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Tue Mar 6 23:17:08 2012
@@ -20,6 +20,7 @@ package org.apache.solr.cloud;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -35,11 +36,13 @@ import org.apache.solr.common.cloud.Clou
import org.apache.solr.common.cloud.CoreState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.CoreDescriptor;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.Code;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.data.Stat;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -51,25 +54,36 @@ public class OverseerTest extends SolrTe
private static final boolean DEBUG = false;
- private static class MockZKController{
+ public static class MockZKController{
private final SolrZkClient zkClient;
+ private final ZkStateReader zkStateReader;
private final String nodeName;
+ private final String collection;
+ private final LeaderElector elector;
+ private final Map<String, CoreState> coreStates = Collections.synchronizedMap(new HashMap<String, CoreState>());
+ private final Map<String, ElectionContext> electionContext = Collections.synchronizedMap(new HashMap<String, ElectionContext>());
- public MockZKController(String zkAddress, String nodeName) throws InterruptedException, TimeoutException, IOException, KeeperException {
+ public MockZKController(String zkAddress, String nodeName, String collection) throws InterruptedException, TimeoutException, IOException, KeeperException {
this.nodeName = nodeName;
+ this.collection = collection;
zkClient = new SolrZkClient(zkAddress, TIMEOUT);
+ zkStateReader = new ZkStateReader(zkClient);
+ zkStateReader.createClusterStateWatchersAndUpdate();
Overseer.createClientNodes(zkClient, nodeName);
// live node
- final String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1";
+ final String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
zkClient.makePath(nodePath, CreateMode.EPHEMERAL, true);
+ elector = new LeaderElector(zkClient);
}
private void deleteNode(final String path) {
try {
Stat stat = zkClient.exists(path, null, false);
- zkClient.delete(path, stat.getVersion(), false);
+ if (stat != null) {
+ zkClient.delete(path, stat.getVersion(), false);
+ }
} catch (KeeperException e) {
fail("Unexpected KeeperException!" + e);
} catch (InterruptedException e) {
@@ -82,22 +96,72 @@ public class OverseerTest extends SolrTe
deleteNode(ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1");
zkClient.close();
} catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ //e.printStackTrace();
}
}
- public void publishState(String coreName, String stateName, int numShards) throws KeeperException, InterruptedException{
- HashMap<String,String> coreProps = new HashMap<String,String>();
- coreProps.put(ZkStateReader.STATE_PROP, stateName);
- coreProps.put(ZkStateReader.NODE_NAME_PROP, nodeName);
- coreProps.put(ZkStateReader.CORE_NAME_PROP, coreName);
- CoreState state = new CoreState(coreName, "collection1", coreProps, numShards);
+ public void publishState(String coreName, String stateName, int numShards)
+ throws KeeperException, InterruptedException, IOException {
+ if (stateName == null) {
+ coreStates.remove(coreName);
+ ElectionContext ec = electionContext.remove(coreName);
+ if (ec != null) {
+ ec.cancelElection();
+ }
+ } else {
+ HashMap<String,String> coreProps = new HashMap<String,String>();
+ coreProps.put(ZkStateReader.STATE_PROP, stateName);
+ coreProps.put(ZkStateReader.NODE_NAME_PROP, nodeName);
+ coreProps.put(ZkStateReader.CORE_NAME_PROP, coreName);
+ coreProps.put(ZkStateReader.COLLECTION_PROP, collection);
+ coreProps.put(ZkStateReader.BASE_URL_PROP, "http://" + nodeName
+ + "/solr/");
+ CoreState state = new CoreState(coreName, collection, coreProps,
+ numShards);
+ coreStates.remove(coreName);
+ coreStates.put(coreName, state);
+ }
final String statePath = Overseer.STATES_NODE + "/" + nodeName;
- zkClient.setData(statePath, ZkStateReader.toJSON(new CoreState[] {state}), true);
+ zkClient.setData(
+ statePath,
+ ZkStateReader.toJSON(coreStates.values().toArray(
+ new CoreState[coreStates.size()])), true);
+
+ for (int i = 0; i < 10; i++) {
+ String shardId = getShardId(coreName);
+ if (shardId != null) {
+ try {
+ zkClient.makePath("/collections/" + collection + "/leader_elect/"
+ + shardId + "/election", true);
+ } catch (NodeExistsException nee) {}
+ ZkNodeProps props = new ZkNodeProps(ZkStateReader.BASE_URL_PROP,
+ "http://" + nodeName + "/solr/", ZkStateReader.NODE_NAME_PROP,
+ nodeName, ZkStateReader.CORE_NAME_PROP, coreName,
+ ZkStateReader.SHARD_ID_PROP, shardId,
+ ZkStateReader.COLLECTION_PROP, collection);
+ ShardLeaderElectionContextBase ctx = new ShardLeaderElectionContextBase(
+ elector, shardId, collection, nodeName + "_" + coreName, props,
+ zkStateReader);
+ elector.joinElection(ctx);
+ break;
+ }
+ Thread.sleep(200);
+ }
}
- }
+ private String getShardId(final String coreName) {
+ Map<String,Slice> slices = zkStateReader.getCloudState().getSlices(
+ collection);
+ if (slices != null) {
+ for (Slice slice : slices.values()) {
+ if (slice.getShards().containsKey(nodeName + "_" + coreName))
+ ;
+ return slice.getName();
+ }
+ }
+ return null;
+ }
+ }
@BeforeClass
public static void beforeClass() throws Exception {
@@ -126,7 +190,10 @@ public class OverseerTest extends SolrTe
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+ zkClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true);
+
ZkStateReader reader = new ZkStateReader(zkClient);
+ reader.createClusterStateWatchersAndUpdate();
System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "3");
@@ -151,7 +218,7 @@ public class OverseerTest extends SolrTe
collection1Desc.setCollectionName("collection1");
CoreDescriptor desc1 = new CoreDescriptor(null, "core" + (i + 1), "");
desc1.setCloudDescriptor(collection1Desc);
- zkController.preRegisterSetup(null, desc1);
+ zkController.preRegister(desc1);
ids[i] = zkController.register("core" + (i + 1), desc1);
}
@@ -170,6 +237,8 @@ public class OverseerTest extends SolrTe
assertNotNull(reader.getLeaderUrl("collection1", "shard3", 15000));
} finally {
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ System.clearProperty("bootstrap_confdir");
if (DEBUG) {
if (zkController != null) {
zkClient.printLayoutToStdOut();
@@ -183,8 +252,6 @@ public class OverseerTest extends SolrTe
}
server.shutdown();
}
-
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
}
@Test
@@ -208,8 +275,11 @@ public class OverseerTest extends SolrTe
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+ zkClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true);
+
reader = new ZkStateReader(zkClient);
-
+ reader.createClusterStateWatchersAndUpdate();
+
System.setProperty(ZkStateReader.NUM_SHARDS_PROP, Integer.valueOf(sliceCount).toString());
for (int i = 0; i < nodeCount; i++) {
@@ -248,7 +318,7 @@ public class OverseerTest extends SolrTe
final CoreDescriptor desc = new CoreDescriptor(null, coreName, "");
desc.setCloudDescriptor(collection1Desc);
try {
- controllers[slot % nodeCount].preRegisterSetup(null, desc);
+ controllers[slot % nodeCount].preRegister(desc);
ids[slot] = controllers[slot % nodeCount]
.register(coreName, desc);
} catch (Throwable e) {
@@ -324,6 +394,8 @@ public class OverseerTest extends SolrTe
}
} finally {
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ System.clearProperty("bootstrap_confdir");
if (DEBUG) {
if (controllers[0] != null) {
zkClient.printLayoutToStdOut();
@@ -344,8 +416,6 @@ public class OverseerTest extends SolrTe
nodeExecutors[i].shutdownNow();
}
}
-
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
}
//wait until collections are available
@@ -464,6 +534,21 @@ public class OverseerTest extends SolrTe
fail("Illegal state, was:" + coreState + " expected:" + expectedState + "cloudState:" + reader.getCloudState());
}
+ private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore) throws InterruptedException, KeeperException {
+ int maxIterations = 100;
+ while(maxIterations-->0) {
+ ZkNodeProps props = reader.getCloudState().getLeader(collection, shard);
+ if(props!=null) {
+ if(expectedCore.equals(props.get(ZkStateReader.CORE_NAME_PROP))) {
+ return;
+ }
+ }
+ Thread.sleep(100);
+ }
+
+ assertEquals("Unexpected shard leader coll:" + collection + " shard:" + shard, expectedCore, (reader.getCloudState().getLeader(collection, shard)!=null)?reader.getCloudState().getLeader(collection, shard).get(ZkStateReader.CORE_NAME_PROP):null);
+ }
+
@Test
public void testOverseerFailure() throws Exception {
String zkDir = dataDir.getAbsolutePath() + File.separator
@@ -485,7 +570,7 @@ public class OverseerTest extends SolrTe
reader = new ZkStateReader(controllerClient);
reader.createClusterStateWatchersAndUpdate();
- mockController = new MockZKController(server.getZkAddress(), "node1");
+ mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
overseerClient = electNewOverseer(server.getZkAddress());
@@ -519,6 +604,12 @@ public class OverseerTest extends SolrTe
.getLiveNodes().size());
assertEquals("Shard count does not match", 1, reader.getCloudState()
.getSlice("collection1", "shard1").getShards().size());
+ version = getCloudStateVersion(controllerClient);
+ mockController.publishState("core1", null,1);
+ while(version == getCloudStateVersion(controllerClient));
+ Thread.sleep(100);
+ assertEquals("Shard count does not match", 0, reader.getCloudState()
+ .getSlice("collection1", "shard1").getShards().size());
} finally {
if (mockController != null) {
@@ -537,8 +628,112 @@ public class OverseerTest extends SolrTe
server.shutdown();
}
}
+
+ private class OverseerRestarter implements Runnable{
+ SolrZkClient overseerClient = null;
+ public volatile boolean run = true;
+ private final String zkAddress;
+
+ public OverseerRestarter(String zkAddress) {
+ this.zkAddress = zkAddress;
+ }
+
+ @Override
+ public void run() {
+ try {
+ overseerClient = electNewOverseer(zkAddress);
+ } catch (Throwable t) {
+ //t.printStackTrace();
+ }
+ while (run) {
+ if(random.nextInt(20)==1){
+ try {
+ overseerClient.close();
+ overseerClient = electNewOverseer(zkAddress);
+ } catch (Throwable e) {
+ //e.printStackTrace();
+ }
+ }
+ try {
+ Thread.sleep(100);
+ } catch (Throwable e) {
+ //e.printStackTrace();
+ }
+ }
+ try {
+ overseerClient.close();
+ } catch (Throwable e) {
+ //e.printStackTrace();
+ }
+ }
+ }
+
@Test
+ public void testShardLeaderChange() throws Exception {
+ String zkDir = dataDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+ final ZkTestServer server = new ZkTestServer(zkDir);
+ SolrZkClient controllerClient = null;
+ ZkStateReader reader = null;
+ MockZKController mockController = null;
+ MockZKController mockController2 = null;
+ OverseerRestarter killer = null;
+ Thread killerThread = null;
+ try {
+ server.run();
+ controllerClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+ AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+ controllerClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true);
+
+ killer = new OverseerRestarter(server.getZkAddress());
+ killerThread = new Thread(killer);
+ killerThread.start();
+
+ reader = new ZkStateReader(controllerClient);
+ reader.createClusterStateWatchersAndUpdate();
+
+ for (int i = 0; i < 20; i++) {
+ mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
+ mockController.publishState("core1", "state1",1);
+ if(mockController2!=null) {
+ mockController2.close();
+ mockController2 = null;
+ }
+ mockController.publishState("core1", "state2",1);
+ mockController2 = new MockZKController(server.getZkAddress(), "node2", "collection1");
+ mockController.publishState("core1", "state1",1);
+ verifyShardLeader(reader, "collection1", "shard1", "core1");
+ mockController2.publishState("core4", "state2" ,1);
+ mockController.close();
+ mockController = null;
+ verifyShardLeader(reader, "collection1", "shard1", "core4");
+ }
+ } finally {
+ if (killer != null) {
+ killer.run = false;
+ if (killerThread != null) {
+ killerThread.join();
+ }
+ }
+ if (mockController != null) {
+ mockController.close();
+ }
+ if (mockController2 != null) {
+ mockController2.close();
+ }
+ if (controllerClient != null) {
+ controllerClient.close();
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ server.shutdown();
+ }
+ }
+
+ @Test
public void testDoubleAssignment() throws Exception {
String zkDir = dataDir.getAbsolutePath() + File.separator
+ "zookeeper/server1/data";
@@ -561,7 +756,7 @@ public class OverseerTest extends SolrTe
reader = new ZkStateReader(controllerClient);
reader.createClusterStateWatchersAndUpdate();
- mockController = new MockZKController(server.getZkAddress(), "node1");
+ mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
overseerClient = electNewOverseer(server.getZkAddress());
@@ -575,7 +770,7 @@ public class OverseerTest extends SolrTe
int version = getCloudStateVersion(controllerClient);
- mockController = new MockZKController(server.getZkAddress(), "node1");
+ mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
while (version == getCloudStateVersion(controllerClient));
@@ -593,7 +788,6 @@ public class OverseerTest extends SolrTe
}
assertEquals("Shard was found in more than 1 times in CloudState", 1,
numFound);
-
} finally {
if (overseerClient != null) {
overseerClient.close();
@@ -635,7 +829,7 @@ public class OverseerTest extends SolrTe
reader = new ZkStateReader(controllerClient);
reader.createClusterStateWatchersAndUpdate();
- mockController = new MockZKController(server.getZkAddress(), "node1");
+ mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
overseerClient = electNewOverseer(server.getZkAddress());
@@ -676,7 +870,7 @@ public class OverseerTest extends SolrTe
LeaderElector overseerElector = new LeaderElector(zkClient);
ElectionContext ec = new OverseerElectionContext(address.replaceAll("/", "_"), zkClient, reader);
overseerElector.setup(ec);
- overseerElector.joinElection(ec, null);
+ overseerElector.joinElection(ec);
return zkClient;
}
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Tue Mar 6 23:17:08 2012
@@ -22,12 +22,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import junit.framework.Assert;
+
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrConfig;
import org.apache.zookeeper.CreateMode;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -55,6 +56,7 @@ public class ZkControllerTest extends So
try {
server.run();
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
@@ -142,6 +144,103 @@ public class ZkControllerTest extends So
}
+ @Test
+ public void testCoreUnload() throws Exception {
+
+ String zkDir = dataDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+
+ ZkTestServer server = new ZkTestServer(zkDir);
+
+ ZkController zkController = null;
+ SolrZkClient zkClient = null;
+ try {
+ server.run();
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+ AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+
+ zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+ zkClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true);
+
+ ZkStateReader reader = new ZkStateReader(zkClient);
+ reader.createClusterStateWatchersAndUpdate();
+
+ System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "1");
+ System.setProperty("solrcloud.skip.autorecovery", "true");
+
+ zkController = new ZkController(null, server.getZkAddress(), TIMEOUT,
+ 10000, "localhost", "8983", "solr",
+ new CurrentCoreDescriptorProvider() {
+
+ @Override
+ public List<CoreDescriptor> getCurrentDescriptors() {
+ // do nothing
+ return null;
+ }
+ });
+
+ System.setProperty("bootstrap_confdir", getFile("solr/conf")
+ .getAbsolutePath());
+
+ final int numShards = 2;
+ final String[] ids = new String[numShards];
+
+ for (int i = 0; i < numShards; i++) {
+ CloudDescriptor collection1Desc = new CloudDescriptor();
+ collection1Desc.setCollectionName("collection1");
+ CoreDescriptor desc1 = new CoreDescriptor(null, "core" + (i + 1), "");
+ desc1.setCloudDescriptor(collection1Desc);
+ zkController.preRegister(desc1);
+ ids[i] = zkController.register("core" + (i + 1), desc1);
+ }
+
+ assertEquals("shard1", ids[0]);
+ assertEquals("shard1", ids[1]);
+
+ assertNotNull(reader.getLeaderUrl("collection1", "shard1", 15000));
+
+ assertEquals("Shard(s) missing from cloudstate", 2, zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size());
+
+ // unregister current leader
+ final ZkNodeProps shard1LeaderProps = reader.getLeaderProps(
+ "collection1", "shard1");
+ final String leaderUrl = reader.getLeaderUrl("collection1", "shard1",
+ 15000);
+
+ final CloudDescriptor collection1Desc = new CloudDescriptor();
+ collection1Desc.setCollectionName("collection1");
+ final CoreDescriptor desc1 = new CoreDescriptor(null,
+ shard1LeaderProps.get(ZkStateReader.CORE_NAME_PROP), "");
+ desc1.setCloudDescriptor(collection1Desc);
+ zkController.unregister(
+ shard1LeaderProps.get(ZkStateReader.CORE_NAME_PROP), collection1Desc);
+ assertNotSame(
+ "New leader was not promoted after unregistering the current leader.",
+ leaderUrl, reader.getLeaderUrl("collection1", "shard1", 15000));
+ assertNotNull("New leader was null.",
+ reader.getLeaderUrl("collection1", "shard1", 15000));
+
+ Thread.sleep(2000);
+ assertEquals("shard was not unregistered", 1, zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size());
+ } finally {
+ System.clearProperty("solrcloud.skip.autorecovery");
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
+ System.clearProperty("bootstrap_confdir");
+ if (DEBUG) {
+ if (zkController != null) {
+ zkClient.printLayoutToStdOut();
+ }
+ }
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ if (zkController != null) {
+ zkController.close();
+ }
+ server.shutdown();
+ }
+ }
+
@Override
public void tearDown() throws Exception {
super.tearDown();
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java Tue Mar 6 23:17:08 2012
@@ -20,6 +20,7 @@ package org.apache.solr.cloud;
import java.io.File;
import java.util.concurrent.atomic.AtomicInteger;
+import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -91,7 +92,7 @@ public class ZkSolrClientTest extends Ab
try {
zkClient.makePath("collections/collection2", false);
- TestCase.fail("Server should be down here");
+ Assert.fail("Server should be down here");
} catch (KeeperException.ConnectionLossException e) {
}
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java Tue Mar 6 23:17:08 2012
@@ -235,7 +235,7 @@ public class ZkTestServer {
} catch(IllegalStateException e) {
}
- if (cnt == 100) {
+ if (cnt == 500) {
throw new RuntimeException("Could not get the port for ZooKeeper server");
}
cnt++;
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java Tue Mar 6 23:17:08 2012
@@ -71,7 +71,7 @@ public class TestArbitraryIndexDir exten
+ System.getProperty("file.separator") + "data");
dataDir.mkdirs();
- solrConfig = h.createConfig("solrconfig.xml");
+ solrConfig = TestHarness.createConfig("solrconfig.xml");
h = new TestHarness( dataDir.getAbsolutePath(),
solrConfig,
"schema12.xml");
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Tue Mar 6 23:17:08 2012
@@ -35,6 +35,7 @@ import org.apache.lucene.search.MatchAll
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.TestDistributedSearch;
import org.apache.solr.client.solrj.SolrServer;
@@ -389,7 +390,7 @@ public class TestReplicationHandler exte
assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
- String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
+ String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
//start config files replication test
@@ -447,7 +448,7 @@ public class TestReplicationHandler exte
assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
- String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
+ String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
// start stop polling test
@@ -527,7 +528,7 @@ public class TestReplicationHandler exte
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
- String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
+ String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
System.out.println("replicate slave to master");
@@ -599,7 +600,7 @@ public class TestReplicationHandler exte
assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
- String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
+ String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
// NOTE: the master only replicates after startup now!
@@ -654,7 +655,7 @@ public class TestReplicationHandler exte
assertEquals(10, slaveQueryResult.getNumFound());
//compare results
- String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
+ String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
Object version = getIndexVersion(masterClient).get("indexversion");
@@ -714,7 +715,7 @@ public class TestReplicationHandler exte
assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
- String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
+ String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
//start config files replication test
@@ -769,8 +770,17 @@ public class TestReplicationHandler exte
private void doTestBackup() throws Exception {
+ String configFile = "solrconfig-master1.xml";
+ boolean addNumberToKeepInRequest = true;
+ String backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM;
+ if(random.nextBoolean()) {
+ configFile = "solrconfig-master1-keepOneBackup.xml";
+ addNumberToKeepInRequest = false;
+ backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_INIT_PARAM;
+ }
+
masterJetty.stop();
- master.copyConfigFile(CONF_DIR + "solrconfig-master1.xml",
+ master.copyConfigFile(CONF_DIR + configFile,
"solrconfig.xml");
masterJetty = createJetty(master);
@@ -785,9 +795,17 @@ public class TestReplicationHandler exte
class BackupThread extends Thread {
volatile String fail = null;
+ final boolean addNumberToKeepInRequest;
+ String backupKeepParamName;
+ BackupThread(boolean addNumberToKeepInRequest, String backupKeepParamName) {
+ this.addNumberToKeepInRequest = addNumberToKeepInRequest;
+ this.backupKeepParamName = backupKeepParamName;
+ }
@Override
public void run() {
- String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP + "&" + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + "=1";
+ String masterUrl =
+ "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP +
+ (addNumberToKeepInRequest ? "&" + backupKeepParamName + "=1" : "");
URL url;
InputStream stream = null;
try {
@@ -846,7 +864,7 @@ public class TestReplicationHandler exte
File[] snapDir = new File[2];
String firstBackupTimestamp = null;
for(int i=0 ; i<2 ; i++) {
- BackupThread backupThread = new BackupThread();
+ BackupThread backupThread = new BackupThread(addNumberToKeepInRequest, backupKeepParamName);
backupThread.start();
File dataDir = new File(master.getDataDir());
@@ -896,7 +914,7 @@ public class TestReplicationHandler exte
dir.close();
}
if(snapDir[0].exists()) {
- fail("The first backup should have been cleaned up because " + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + " was set to 1");
+ fail("The first backup should have been cleaned up because " + backupKeepParamName + " was set to 1.");
}
for(int i=0 ; i< snapDir.length ; i++) {