You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [35/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/...
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java Fri Jan 18 18:30:54 2013
@@ -43,6 +43,7 @@ public class TestDistributedGrouping ext
String tdate_b = "b_n_tdt";
String oddField="oddField_s";
+ @Override
public void doTest() throws Exception {
del("*:*");
commit();
@@ -50,6 +51,8 @@ public class TestDistributedGrouping ext
handle.clear();
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
+ handle.put("grouped", UNORDERED); // distrib grouping doesn't guarantee order of top level group commands
+
// Test distributed grouping with empty indices
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " asc, id asc");
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " asc, id asc", "hl","true","hl.fl",t1);
@@ -162,6 +165,30 @@ public class TestDistributedGrouping ext
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc");
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc");
+ // SOLR-4150: what if group.query has no matches,
+ // or only matches on one shard
+ query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "group.query", "id:5", // single doc, so only one shard will have it
+ "group.limit", 10, "sort", i1 + " asc, id asc");
+ handle.put(t1 + ":this_will_never_match", SKIP); // :TODO: SOLR-4181
+ query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "group.query", t1 + ":this_will_never_match",
+ "group.limit", 10, "sort", i1 + " asc, id asc");
+
+ // SOLR-4164: main query matches nothing, or only matches on one shard
+ query("q", "bogus_s:nothing", // no docs match
+ "group", "true",
+ "group.query", t1 + ":this_will_never_match",
+ "group.field", i1,
+ "fl", "id", "group.limit", "2", "group.format", "simple");
+ query("q", "id:5", // one doc matches, so only one shard
+ "rows", 100, "fl", "id," + i1, "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "group.field", i1,
+ "group.limit", 10, "sort", i1 + " asc, id asc");
+
// SOLR-3109
query("q", t1 + ":eggs", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", tlong + " asc, id asc");
query("q", i1 + ":232", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", tlong + " asc, id asc");
@@ -179,6 +206,19 @@ public class TestDistributedGrouping ext
query("q", "*:*", "fq", s1 + ":a", "fl", "id," + i1, "group", "true", "group.field", i1, "sort", i1 + " asc, id asc", "group.ngroups", "true");
query("q", "*:*", "fq", s1 + ":a", "rows", 0, "fl", "id," + i1, "group", "true", "group.field", i1, "sort", i1 + " asc, id asc", "group.ngroups", "true");
+ // SOLR-3960 - include a postfilter
+ for (String facet : new String[] { "false", "true"}) {
+ for (String fcache : new String[] { "", " cache=false cost=200"}) {
+ query("q", "*:*", "rows", 100, "fl", "id," + i1,
+ "group.limit", 10, "sort", i1 + " asc, id asc",
+ "group", "true", "group.field", i1,
+ "fq", "{!frange l=50 "+fcache+"}"+tlong,
+ "facet.field", t1,
+ "facet", facet
+ );
+ }
+ }
+
ModifiableSolrParams params = new ModifiableSolrParams();
Object[] q = {"q", "*:*", "fq", s1 + ":a", "rows", 1, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "group.ngroups", "true"};
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Fri Jan 18 18:30:54 2013
@@ -502,6 +502,18 @@ public class TestGroupingSearch extends
"'doclist':{'numFound':4,'start':0,'docs':[{'id':'3'},{'id':'4'},{'id':'2'}]}}}"
);
+ // group.query that matches nothing
+ assertJQ(req("fq",filt,
+ "q","{!func}"+f2,
+ "group","true",
+ "group.query","id:[2 TO 5]",
+ "group.query","id:1000",
+ "fl","id",
+ "group.limit","3")
+ ,"/grouped/id:[2 TO 5]=={'matches':10,'doclist':{'numFound':4,'start':0,'docs':[{'id':'3'},{'id':'4'},{'id':'2'}]}}"
+ ,"/grouped/id:1000=={'matches':10,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ );
+
// group.query and offset
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.query","id:[2 TO 5]", "fl","id", "group.limit","3", "group.offset","2")
,"/grouped=={'id:[2 TO 5]':{'matches':10," +
@@ -573,6 +585,34 @@ public class TestGroupingSearch extends
, "/grouped/foo_i=={'matches':10,'doclist':"
+ "{'numFound':10,'start':1,'docs':[{'id':'10'},{'id':'3'},{'id':'6'}]}}"
);
+
+ //////////////////////// grouping where main query matches nothing
+ assertJQ(req("fq", filt, "q", "bogus_s:nothing", "group", "true", "group.field", f, "fl", "id", "group.limit", "2", "group.format", "simple")
+ , "/grouped/foo_i=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ );
+ assertJQ(req("fq",filt, "q","bogus_s:nothing", "group","true",
+ "group.query","id:[2 TO 5]",
+ "group.query","id:[5 TO 5]",
+ "group.field",f,
+ "rows","1",
+ "fl","id", "group.limit","2")
+ ,"/grouped/id:[2 TO 5]=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ ,"/grouped/id:[5 TO 5]=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ ,"/grouped/"+f+"=={'matches':0,'groups':[]}"
+ );
+ assertJQ(req("fq",filt,
+ "q","bogus_s:nothing",
+ "group","true",
+ "group.query","id:[2 TO 5]",
+ "group.query","id:1000",
+ "fl","id",
+ "group.limit","3")
+ ,"/grouped/id:[2 TO 5]=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ ,"/grouped/id:1000=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ );
+
+
+
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java Fri Jan 18 18:30:54 2013
@@ -16,7 +16,6 @@
*/
package org.apache.solr;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.util.AbstractSolrTestCase;
@@ -37,9 +36,6 @@ import java.util.Properties;
*
* @since solr 1.4
*/
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class
-})
public class TestSolrCoreProperties extends LuceneTestCase {
private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
JettySolrRunner solrJetty;
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestTrie.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestTrie.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/TestTrie.java Fri Jan 18 18:30:54 2013
@@ -16,6 +16,8 @@
*/
package org.apache.solr;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.DateField;
import org.apache.solr.schema.FieldType;
@@ -25,6 +27,7 @@ import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.TimeZone;
@@ -47,6 +50,38 @@ public class TestTrie extends SolrTestCa
clearIndex();
super.tearDown();
}
+
+ @Test
+ public void testTokenizer() throws Exception {
+ FieldType type = h.getCore().getSchema().getFieldType("tint");
+ assertTrue(type instanceof TrieField);
+
+ String value = String.valueOf(random().nextInt());
+ TokenStream ts = type.getAnalyzer().tokenStream("dummy", new StringReader(value));
+ OffsetAttribute ofsAtt = ts.addAttribute(OffsetAttribute.class);
+ ts.reset();
+ int count = 0;
+ while (ts.incrementToken()) {
+ count++;
+ assertEquals(0, ofsAtt.startOffset());
+ assertEquals(value.length(), ofsAtt.endOffset());
+ }
+ final int precStep = ((TrieField) type).getPrecisionStep();
+ assertEquals( (32 + precStep - 1) / precStep, count);
+ ts.end();
+ assertEquals(value.length(), ofsAtt.startOffset());
+ assertEquals(value.length(), ofsAtt.endOffset());
+ ts.close();
+
+ // Test empty one:
+ ts = type.getAnalyzer().tokenStream("dummy", new StringReader(""));
+ ts.reset();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ assertEquals(0, ofsAtt.startOffset());
+ assertEquals(0, ofsAtt.endOffset());
+ ts.close();
+ }
@Test
public void testTrieIntRangeSearch() throws Exception {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Fri Jan 18 18:30:54 2013
@@ -17,13 +17,24 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
@@ -32,7 +43,11 @@ import org.apache.solr.common.cloud.ZkNo
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.ReplicationHandler;
import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.BeforeClass;
/**
@@ -110,7 +125,7 @@ public class BasicDistributedZk2Test ext
// TODO: bring this to it's own method?
// try indexing to a leader that has no replicas up
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
- ZkNodeProps leaderProps = zkStateReader.getLeaderProps(
+ ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(
DEFAULT_COLLECTION, SHARD2);
String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP);
@@ -200,26 +215,6 @@ public class BasicDistributedZk2Test ext
jetties.addAll(shardToJetty.get(SHARD2));
jetties.remove(deadShard);
- // wait till live nodes drops by 1
- int liveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
- int tries = 50;
- while(oldLiveNodes == liveNodes) {
- Thread.sleep(100);
- if (tries-- == 0) {
- fail("We expected a node to drop...");
- }
- liveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
- }
- assertEquals(4, liveNodes);
-
- int cnt = 0;
- for (CloudJettyRunner cjetty : jetties) {
- waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
- .getFilter()).getCores().getZkController().getZkStateReader(),
- deadShard, cnt++);
- }
- waitToSeeNotLive(cloudClient.getZkStateReader(), deadShard);
-
// ensure shard is dead
try {
index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1,
@@ -269,7 +264,14 @@ public class BasicDistributedZk2Test ext
UpdateRequest ureq = new UpdateRequest();
ureq.add(doc);
// ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
- ureq.process(cloudClient);
+
+ try {
+ ureq.process(cloudClient);
+ } catch(SolrServerException e){
+ // try again
+ Thread.sleep(500);
+ ureq.process(cloudClient);
+ }
commit();
@@ -319,14 +321,7 @@ public class BasicDistributedZk2Test ext
// recover over 100 docs so we do more than just peer sync (replicate recovery)
chaosMonkey.stopJetty(deadShard);
-
- for (CloudJettyRunner cjetty : jetties) {
- waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
- .getFilter()).getCores().getZkController().getZkStateReader(),
- deadShard);
- }
- waitToSeeNotLive(cloudClient.getZkStateReader(), deadShard);
-
+
for (int i = 0; i < 226; i++) {
doc = new SolrInputDocument();
doc.addField("id", 2000 + i);
@@ -345,9 +340,110 @@ public class BasicDistributedZk2Test ext
// make sure we have published we are recovering
Thread.sleep(1500);
+ waitForThingsToLevelOut(15);
+
+ Thread.sleep(500);
+
waitForRecoveriesToFinish(false);
checkShardConsistency(true, false);
+
+ // try a backup command
+ final HttpSolrServer client = (HttpSolrServer) shardToJetty.get(SHARD2).get(0).client.solrClient;
+ System.out.println("base url: "+ client.getBaseURL());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("qt", "/replication");
+ params.set("command", "backup");
+
+ QueryRequest request = new QueryRequest(params);
+ NamedList<Object> results = client.request(request );
+ System.out.println("results:" + results);
+
+
+ checkForBackupSuccess(client);
+
+ }
+ private void checkForBackupSuccess(final HttpSolrServer client)
+ throws InterruptedException, IOException {
+ class CheckStatus extends Thread {
+ volatile String fail = null;
+ volatile String response = null;
+ volatile boolean success = false;
+ final Pattern p = Pattern
+ .compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
+
+ CheckStatus() {}
+
+ @Override
+ public void run() {
+ String masterUrl = client.getBaseURL() + "/replication?command="
+ + ReplicationHandler.CMD_DETAILS;
+ URL url;
+ InputStream stream = null;
+ try {
+ url = new URL(masterUrl);
+ stream = url.openStream();
+ response = IOUtils.toString(stream, "UTF-8");
+ if (response.contains("<str name=\"status\">success</str>")) {
+ Matcher m = p.matcher(response);
+ if (!m.find()) {
+ fail("could not find the completed timestamp in response.");
+ }
+
+ success = true;
+ }
+ stream.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail = e.getMessage();
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+
+ };
+ }
+ ;
+ SolrCore core = ((SolrDispatchFilter) shardToJetty.get(SHARD2).get(0).jetty
+ .getDispatchFilter().getFilter()).getCores().getCore("collection1");
+ String ddir;
+ try {
+ ddir = core.getDataDir();
+ } finally {
+ core.close();
+ }
+ File dataDir = new File(ddir);
+
+ int waitCnt = 0;
+ CheckStatus checkStatus = new CheckStatus();
+ while (true) {
+ checkStatus.run();
+ if (checkStatus.fail != null) {
+ fail(checkStatus.fail);
+ }
+ if (checkStatus.success) {
+ break;
+ }
+ Thread.sleep(200);
+ if (waitCnt == 20) {
+ fail("Backup success not detected:" + checkStatus.response);
+ }
+ waitCnt++;
+ }
+
+ File[] files = dataDir.listFiles(new FilenameFilter() {
+
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name.startsWith("snapshot")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(Arrays.asList(files).toString(), 1, files.length);
+ File snapDir = files[0];
+
+ AbstractSolrTestCase.recurseDelete(snapDir); // clean up the snap dir
}
private void addNewReplica() throws Exception {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Fri Jan 18 18:30:54 2013
@@ -23,10 +23,8 @@ import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -38,7 +36,6 @@ import java.util.concurrent.ThreadPoolEx
import java.util.concurrent.TimeUnit;
import org.apache.lucene.util.LuceneTestCase.Slow;
-import org.apache.lucene.util._TestUtil;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
@@ -48,18 +45,16 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
-import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;
import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -164,7 +159,7 @@ public class BasicDistributedZkTest exte
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
// make sure we have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
} // make sure we again have leaders for each shard
waitForRecoveriesToFinish(false);
@@ -339,7 +334,6 @@ public class BasicDistributedZkTest exte
testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");
testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");
- testCollectionsAPI();
testCoreUnloadAndLeaders();
testUnloadLotsOfCores();
testStopAndStartCoresInOneInstance();
@@ -365,7 +359,8 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(0);
String url1 = getBaseUrl(client);
HttpSolrServer server = new HttpSolrServer(url1);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
server.request(createCmd);
createCmd = new Create();
@@ -384,7 +379,7 @@ public class BasicDistributedZkTest exte
server.request(unloadCmd);
// there should be only one shard
- Slice shard2 = solrj.getZkStateReader().getClusterState().getSlice(collection, "shard2");
+ Slice shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
long timeoutAt = System.currentTimeMillis() + 30000;
while (shard2 != null) {
if (System.currentTimeMillis() > timeoutAt) {
@@ -393,12 +388,12 @@ public class BasicDistributedZkTest exte
}
Thread.sleep(50);
- shard2 = solrj.getZkStateReader().getClusterState().getSlice(collection, "shard2");
+ shard2 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard2");
}
- Slice shard1 = solrj.getZkStateReader().getClusterState().getSlice(collection, "shard1");
+ Slice shard1 = getCommonCloudSolrServer().getZkStateReader().getClusterState().getSlice(collection, "shard1");
assertNotNull(shard1);
- assertTrue(solrj.getZkStateReader().getClusterState().getCollections().contains(collection));
+ assertTrue(getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection));
// now unload one of the other
unloadCmd = new Unload(false);
@@ -408,7 +403,7 @@ public class BasicDistributedZkTest exte
//printLayout();
// the collection should be gone
timeoutAt = System.currentTimeMillis() + 30000;
- while (solrj.getZkStateReader().getClusterState().getCollections().contains(collection)) {
+ while (getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collection)) {
if (System.currentTimeMillis() > timeoutAt) {
printLayout();
fail("Still found collection");
@@ -427,6 +422,8 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(0);
String url1 = getBaseUrl(client);
HttpSolrServer server = new HttpSolrServer(url1);
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
Create createCmd = new Create();
createCmd.setCoreName("unloadcollection1");
@@ -436,11 +433,11 @@ public class BasicDistributedZkTest exte
createCmd.setDataDir(core1DataDir);
server.request(createCmd);
- ZkStateReader zkStateReader = solrj.getZkStateReader();
+ ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
zkStateReader.updateClusterState(true);
- int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").size();
+ int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
assertEquals(1, slices);
client = clients.get(1);
@@ -455,7 +452,7 @@ public class BasicDistributedZkTest exte
server.request(createCmd);
zkStateReader.updateClusterState(true);
- slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").size();
+ slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
assertEquals(1, slices);
waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
@@ -529,7 +526,7 @@ public class BasicDistributedZkTest exte
}
// ensure there is a leader
- zkStateReader.getLeaderProps("unloadcollection", "shard1", 15000);
+ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
addClient = new HttpSolrServer(url2 + "/unloadcollection2");
// add a few docs while the leader is down
@@ -571,7 +568,7 @@ public class BasicDistributedZkTest exte
}
}
- zkStateReader.getLeaderProps("unloadcollection", "shard1", 15000);
+ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
// set this back
@@ -617,13 +614,14 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(2);
String url3 = getBaseUrl(client);
final HttpSolrServer server = new HttpSolrServer(url3);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(60000);
ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
- int cnt = atLeast(6);
+ int cnt = random().nextInt(12) + 1;
- // create the 6 cores
+ // create the cores
createCores(server, executor, "multiunload", 2, cnt);
executor.shutdown();
@@ -657,7 +655,8 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(0);
String url3 = getBaseUrl(client);
final HttpSolrServer server = new HttpSolrServer(url3);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
@@ -676,7 +675,7 @@ public class BasicDistributedZkTest exte
ChaosMonkey.start(cloudJettys.get(0).jetty);
cloudClient.getZkStateReader().updateClusterState(true);
try {
- cloudClient.getZkStateReader().getLeaderProps("multiunload2", "shard1", 30000);
+ cloudClient.getZkStateReader().getLeaderRetry("multiunload2", "shard1", 30000);
} catch (SolrException e) {
printLayout();
throw e;
@@ -714,7 +713,6 @@ public class BasicDistributedZkTest exte
}
}
-
private String getBaseUrl(SolrServer client) {
String url2 = ((HttpSolrServer) client).getBaseURL()
.substring(
@@ -724,155 +722,16 @@ public class BasicDistributedZkTest exte
return url2;
}
-
- private void testCollectionsAPI() throws Exception {
-
- // TODO: fragile - because we dont pass collection.confName, it will only
- // find a default if a conf set with a name matching the collection name is found, or
- // if there is only one conf set. That and the fact that other tests run first in this
- // env make this pretty fragile
-
- // create new collections rapid fire
- Map<String,List<Integer>> collectionInfos = new HashMap<String,List<Integer>>();
- int cnt = atLeast(3);
-
- for (int i = 0; i < cnt; i++) {
- createCollection(collectionInfos, i,
- _TestUtil.nextInt(random(), 0, shardCount) + 1,
- _TestUtil.nextInt(random(), 0, 3) + 1);
- }
-
- Set<Entry<String,List<Integer>>> collectionInfosEntrySet = collectionInfos.entrySet();
- for (Entry<String,List<Integer>> entry : collectionInfosEntrySet) {
- String collection = entry.getKey();
- List<Integer> list = entry.getValue();
- checkForCollection(collection, list.get(0));
-
- String url = getUrlFromZk(collection);
-
- HttpSolrServer collectionClient = new HttpSolrServer(url);
-
- // poll for a second - it can take a moment before we are ready to serve
- waitForNon403or404or503(collectionClient);
- }
- ZkStateReader zkStateReader = solrj.getZkStateReader();
- for (int j = 0; j < cnt; j++) {
- waitForRecoveriesToFinish("awholynewcollection_" + j, zkStateReader, false);
- }
-
- List<String> collectionNameList = new ArrayList<String>();
- collectionNameList.addAll(collectionInfos.keySet());
- String collectionName = collectionNameList.get(random().nextInt(collectionNameList.size()));
-
- String url = getUrlFromZk(collectionName);
-
- HttpSolrServer collectionClient = new HttpSolrServer(url);
-
-
- // lets try and use the solrj client to index a couple documents
- SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1,
- "humpty dumpy sat on a wall");
- SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1,
- "humpty dumpy3 sat on a walls");
- SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1,
- "humpty dumpy2 sat on a walled");
-
- collectionClient.add(doc1);
-
- collectionClient.add(doc2);
-
- collectionClient.add(doc3);
-
- collectionClient.commit();
-
- assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
-
- // lets try a collection reload
-
- // get core open times
- Map<String,Long> urlToTimeBefore = new HashMap<String,Long>();
- collectStartTimes(collectionName, urlToTimeBefore);
- assertTrue(urlToTimeBefore.size() > 0);
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.RELOAD.toString());
- params.set("name", collectionName);
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- // we can use this client because we just want base url
- final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(0)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
-
- createNewSolrServer("", baseUrl).request(request);
-
- // reloads make take a short while
- boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore);
- assertTrue("some core start times did not change on reload", allTimesAreCorrect);
-
-
- waitForRecoveriesToFinish("awholynewcollection_" + (cnt - 1), zkStateReader, false);
-
- // remove a collection
- params = new ModifiableSolrParams();
- params.set("action", CollectionAction.DELETE.toString());
- params.set("name", collectionName);
- request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- createNewSolrServer("", baseUrl).request(request);
-
- // ensure its out of the state
- checkForMissingCollection(collectionName);
-
- //collectionNameList.remove(collectionName);
-
- // remove an unknown collection
- params = new ModifiableSolrParams();
- params.set("action", CollectionAction.DELETE.toString());
- params.set("name", "unknown_collection");
- request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- createNewSolrServer("", baseUrl).request(request);
-
- // create another collection should still work
- params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
-
- params.set("numShards", 1);
- params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, 1);
- collectionName = "acollectionafterbaddelete";
-
- params.set("name", collectionName);
- request = new QueryRequest(params);
- request.setPath("/admin/collections");
- createNewSolrServer("", baseUrl).request(request);
-
- checkForCollection(collectionName, 1);
-
- url = getUrlFromZk(collectionName);
-
- collectionClient = new HttpSolrServer(url);
-
- // poll for a second - it can take a moment before we are ready to serve
- waitForNon403or404or503(collectionClient);
-
- for (int j = 0; j < cnt; j++) {
- waitForRecoveriesToFinish(collectionName, zkStateReader, false);
- }
- }
-
-
protected void createCollection(Map<String,List<Integer>> collectionInfos,
- int i, int numShards, int numReplicas) throws SolrServerException, IOException {
+ String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
- params.set("numShards", numShards);
+ params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
- String collectionName = "awholynewcollection_" + i;
+ params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
+ if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
+
int clientIndex = random().nextInt(2);
List<Integer> list = new ArrayList<Integer>();
list.add(numShards);
@@ -882,98 +741,20 @@ public class BasicDistributedZkTest exte
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
- final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
-
- createNewSolrServer("", baseUrl).request(request);
- }
-
- private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
-
-
- long timeoutAt = System.currentTimeMillis() + 30000;
-
- boolean allTimesAreCorrect = false;
- while (System.currentTimeMillis() < timeoutAt) {
- Map<String,Long> urlToTimeAfter = new HashMap<String,Long>();
- collectStartTimes(collectionName, urlToTimeAfter);
+ if (client == null) {
+ final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
+ 0,
+ ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
+ - DEFAULT_COLLECTION.length() - 1);
- boolean retry = false;
- Set<Entry<String,Long>> entries = urlToTimeBefore.entrySet();
- for (Entry<String,Long> entry : entries) {
- Long beforeTime = entry.getValue();
- Long afterTime = urlToTimeAfter.get(entry.getKey());
- assertNotNull(afterTime);
- if (afterTime <= beforeTime) {
- retry = true;
- break;
- }
-
- }
- if (!retry) {
- allTimesAreCorrect = true;
- break;
- }
- }
- return allTimesAreCorrect;
- }
-
- private void collectStartTimes(String collectionName,
- Map<String,Long> urlToTime) throws SolrServerException, IOException {
- Map<String,Map<String,Slice>> collections = solrj.getZkStateReader()
- .getClusterState().getCollectionStates();
- if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName);
-
- Iterator<Entry<String,Slice>> it = slices.entrySet().iterator();
- while (it.hasNext()) {
- Entry<String,Slice> sliceEntry = it.next();
- Map<String,Replica> sliceShards = sliceEntry.getValue().getReplicasMap();
- Iterator<Entry<String,Replica>> shardIt = sliceShards.entrySet()
- .iterator();
- while (shardIt.hasNext()) {
- Entry<String,Replica> shardEntry = shardIt.next();
- ZkCoreNodeProps coreProps = new ZkCoreNodeProps(shardEntry.getValue());
- CoreAdminResponse mcr = CoreAdminRequest.getStatus(
- coreProps.getCoreName(),
- new HttpSolrServer(coreProps.getBaseUrl()));
- long before = mcr.getStartTime(coreProps.getCoreName()).getTime();
- urlToTime.put(coreProps.getCoreUrl(), before);
- }
- }
+ createNewSolrServer("", baseUrl).request(request);
} else {
- throw new IllegalArgumentException("Could not find collection in :"
- + collections.keySet());
+ client.request(request);
}
}
-
- private String getUrlFromZk(String collection) {
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
- Map<String,Slice> slices = clusterState.getCollectionStates().get(collection);
-
- if (slices == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
- }
-
- for (Map.Entry<String,Slice> entry : slices.entrySet()) {
- Slice slice = entry.getValue();
- Map<String,Replica> shards = slice.getReplicasMap();
- Set<Map.Entry<String,Replica>> shardEntries = shards.entrySet();
- for (Map.Entry<String,Replica> shardEntry : shardEntries) {
- final ZkNodeProps node = shardEntry.getValue();
- if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
- return new ZkCoreNodeProps(node).getCoreUrl();
- }
- }
- }
-
- throw new RuntimeException("Could not find a live node for collection:" + collection);
- }
private ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
ZkNodeProps leader = clusterState.getLeader(collection, slice);
if (leader == null) {
throw new RuntimeException("Could not find leader:" + collection + " " + slice);
@@ -981,70 +762,76 @@ public class BasicDistributedZkTest exte
return new ZkCoreNodeProps(leader);
}
- private void waitForNon403or404or503(HttpSolrServer collectionClient)
- throws Exception {
- SolrException exp = null;
- long timeoutAt = System.currentTimeMillis() + 30000;
+ private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
- while (System.currentTimeMillis() < timeoutAt) {
- boolean missing = false;
-
- try {
- collectionClient.query(new SolrQuery("*:*"));
- } catch (SolrException e) {
- if (!(e.code() == 403 || e.code() == 503 || e.code() == 404)) {
- throw e;
- }
- exp = e;
- missing = true;
+ int expectedSlices = numShardsNumReplicaList.get(0);
+ // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
+ int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
+ int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
+
+ Map<String,DocCollection> collections = clusterState
+ .getCollectionStates();
+ if (collections.containsKey(collectionName)) {
+ Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
+ // did we find expectedSlices slices/shards?
+ if (slices.size() != expectedSlices) {
+ return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
}
- if (!missing) {
- return;
+ int totalShards = 0;
+ for (String sliceName : slices.keySet()) {
+ for (Replica replica : slices.get(sliceName).getReplicas()) {
+ if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
+ return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
+ }
+ }
+ totalShards += slices.get(sliceName).getReplicas().size();
}
- Thread.sleep(50);
+ if (totalShards != expectedTotalShards) {
+ return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
+ }
+ return null;
+ } else {
+ return "Could not find new collection " + collectionName;
}
-
- fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
}
-
- private void checkForCollection(String collectionName, int expectedSlices)
+
+ private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
throws Exception {
// check for an expectedSlices new collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 60000;
- boolean found = false;
- boolean sliceMatch = false;
+ long timeoutAt = System.currentTimeMillis() + 120000;
+ boolean success = false;
+ String checkResult = "Didnt get to perform a single check";
while (System.currentTimeMillis() < timeoutAt) {
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
- Map<String,Map<String,Slice>> collections = clusterState
- .getCollectionStates();
- if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName);
- // did we find expectedSlices slices/shards?
- if (slices.size() == expectedSlices) {
- sliceMatch = true;
-
- }
- found = true;
+ checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
+ if (checkResult == null) {
+ success = true;
break;
}
Thread.sleep(500);
}
- if (!found) {
- if (!sliceMatch) {
- fail("Could not find new " + expectedSlices + " slice collection called " + collectionName);
+ if (!success) {
+ super.printLayout();
+ fail(checkResult);
}
}
+
+ private void checkCollectionIsNotCreated(String collectionName)
+ throws Exception {
+ // TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
+ Thread.sleep(10000);
+ assertFalse(collectionName + " not supposed to exist", getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collectionName));
}
private void checkForMissingCollection(String collectionName)
throws Exception {
// check for a collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 15000;
+ long timeoutAt = System.currentTimeMillis() + 30000;
boolean found = true;
while (System.currentTimeMillis() < timeoutAt) {
- solrj.getZkStateReader().updateClusterState(true);
- ClusterState clusterState = solrj.getZkStateReader().getClusterState();
- Map<String,Map<String,Slice>> collections = clusterState
+ getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+ Map<String,DocCollection> collections = clusterState
.getCollectionStates();
if (!collections.containsKey(collectionName)) {
found = false;
@@ -1107,6 +894,7 @@ public class BasicDistributedZkTest exte
// cloud level test mainly needed just to make sure that versions and errors are propagated correctly
private void doOptimisticLockingAndUpdating() throws Exception {
+ log.info("### STARTING doOptimisticLockingAndUpdating");
printLayout();
SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);
@@ -1147,6 +935,7 @@ public class BasicDistributedZkTest exte
private void testNumberOfCommitsWithCommitAfterAdd()
throws SolrServerException, IOException {
+ log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd");
long startCommits = getNumCommits((HttpSolrServer) clients.get(0));
ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update");
@@ -1163,6 +952,8 @@ public class BasicDistributedZkTest exte
private Long getNumCommits(HttpSolrServer solrServer) throws
SolrServerException, IOException {
HttpSolrServer server = new HttpSolrServer(solrServer.getBaseURL());
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/admin/mbeans?key=updateHandler&stats=true");
// use generic request to avoid extra processing of queries
@@ -1178,6 +969,7 @@ public class BasicDistributedZkTest exte
}
private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
+ log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");
System.clearProperty("numShards");
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
@@ -1204,9 +996,9 @@ public class BasicDistributedZkTest exte
// no one should be recovering
- waitForRecoveriesToFinish(oneInstanceCollection2, solrj.getZkStateReader(), false, true);
+ waitForRecoveriesToFinish(oneInstanceCollection2, getCommonCloudSolrServer().getZkStateReader(), false, true);
- assertAllActive(oneInstanceCollection2, solrj.getZkStateReader());
+ assertAllActive(oneInstanceCollection2, getCommonCloudSolrServer().getZkStateReader());
//printLayout();
@@ -1227,7 +1019,7 @@ public class BasicDistributedZkTest exte
query.set("collection", oneInstanceCollection2);
query.set("distrib", true);
- long allDocs = solrj.query(query).getResults().getNumFound();
+ long allDocs = getCommonCloudSolrServer().query(query).getResults().getNumFound();
// System.out.println("1:" + oneDocs);
// System.out.println("2:" + twoDocs);
@@ -1241,15 +1033,15 @@ public class BasicDistributedZkTest exte
assertEquals(3, allDocs);
// we added a role of none on these creates - check for it
- ZkStateReader zkStateReader = solrj.getZkStateReader();
+ ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
zkStateReader.updateClusterState(true);
- Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(oneInstanceCollection2);
+ Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(oneInstanceCollection2);
assertNotNull(slices);
String roles = slices.get("slice1").getReplicasMap().values().iterator().next().getStr(ZkStateReader.ROLES_PROP);
assertEquals("none", roles);
- ZkCoreNodeProps props = new ZkCoreNodeProps(solrj.getZkStateReader().getClusterState().getLeader(oneInstanceCollection2, "slice1"));
+ ZkCoreNodeProps props = new ZkCoreNodeProps(getCommonCloudSolrServer().getZkStateReader().getClusterState().getLeader(oneInstanceCollection2, "slice1"));
// now test that unloading a core gets us a new leader
HttpSolrServer server = new HttpSolrServer(baseUrl);
@@ -1271,6 +1063,7 @@ public class BasicDistributedZkTest exte
}
private void testSearchByCollectionName() throws SolrServerException {
+ log.info("### STARTING testSearchByCollectionName");
SolrServer client = clients.get(0);
final String baseUrl = ((HttpSolrServer) client).getBaseURL().substring(
0,
@@ -1286,6 +1079,7 @@ public class BasicDistributedZkTest exte
}
private void testANewCollectionInOneInstance() throws Exception {
+ log.info("### STARTING testANewCollectionInOneInstance");
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
otherCollectionClients.put(oneInstanceCollection , collectionClients);
@@ -1310,8 +1104,8 @@ public class BasicDistributedZkTest exte
SolrServer client3 = createNewSolrServer(oneInstanceCollection + "3", baseUrl);
SolrServer client4 = createNewSolrServer(oneInstanceCollection + "4", baseUrl);
- waitForRecoveriesToFinish(oneInstanceCollection, solrj.getZkStateReader(), false);
- assertAllActive(oneInstanceCollection, solrj.getZkStateReader());
+ waitForRecoveriesToFinish(oneInstanceCollection, getCommonCloudSolrServer().getZkStateReader(), false);
+ assertAllActive(oneInstanceCollection, getCommonCloudSolrServer().getZkStateReader());
client2.add(getDoc(id, "1"));
client3.add(getDoc(id, "2"));
@@ -1327,7 +1121,7 @@ public class BasicDistributedZkTest exte
query.set("collection", oneInstanceCollection);
query.set("distrib", true);
- long allDocs = solrj.query(query).getResults().getNumFound();
+ long allDocs = getCommonCloudSolrServer().query(query).getResults().getNumFound();
// System.out.println("1:" + oneDocs);
// System.out.println("2:" + twoDocs);
@@ -1347,11 +1141,13 @@ public class BasicDistributedZkTest exte
List<SolrServer> collectionClients, final String baseUrl, final int num,
final String shardId) {
Callable call = new Callable() {
+ @Override
public Object call() {
HttpSolrServer server;
try {
server = new HttpSolrServer(baseUrl);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
Create createCmd = new Create();
createCmd.setRoles("none");
createCmd.setCoreName(collection + num);
@@ -1380,6 +1176,7 @@ public class BasicDistributedZkTest exte
}
private void testMultipleCollections() throws Exception {
+ log.info("### STARTING testMultipleCollections");
// create another 2 collections and search across them
createNewCollection("collection2");
createNewCollection("collection3");
@@ -1394,21 +1191,21 @@ public class BasicDistributedZkTest exte
indexDoc("collection2", getDoc(id, "10000000"));
indexDoc("collection2", getDoc(id, "10000001"));
indexDoc("collection2", getDoc(id, "10000003"));
- solrj.setDefaultCollection("collection2");
- solrj.add(getDoc(id, "10000004"));
- solrj.setDefaultCollection(null);
+ getCommonCloudSolrServer().setDefaultCollection("collection2");
+ getCommonCloudSolrServer().add(getDoc(id, "10000004"));
+ getCommonCloudSolrServer().setDefaultCollection(null);
indexDoc("collection3", getDoc(id, "20000000"));
indexDoc("collection3", getDoc(id, "20000001"));
- solrj.setDefaultCollection("collection3");
- solrj.add(getDoc(id, "10000005"));
- solrj.setDefaultCollection(null);
+ getCommonCloudSolrServer().setDefaultCollection("collection3");
+ getCommonCloudSolrServer().add(getDoc(id, "10000005"));
+ getCommonCloudSolrServer().setDefaultCollection(null);
otherCollectionClients.get("collection2").get(0).commit();
otherCollectionClients.get("collection3").get(0).commit();
- solrj.setDefaultCollection("collection1");
- long collection1Docs = solrj.query(new SolrQuery("*:*")).getResults()
+ getCommonCloudSolrServer().setDefaultCollection("collection1");
+ long collection1Docs = getCommonCloudSolrServer().query(new SolrQuery("*:*")).getResults()
.getNumFound();
long collection2Docs = otherCollectionClients.get("collection2").get(0)
@@ -1429,19 +1226,19 @@ public class BasicDistributedZkTest exte
assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
// try to search multiple with cloud client
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection1Docs + collection2Docs + collection3Docs, found);
query.set("collection", "collection2,collection3");
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection2Docs + collection3Docs, found);
query.set("collection", "collection3");
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection3Docs, found);
query.remove("collection");
- found = solrj.query(query).getResults().getNumFound();
+ found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
assertEquals(collection1Docs, found);
assertEquals(collection3Docs, collection2Docs - 1);
@@ -1473,11 +1270,13 @@ public class BasicDistributedZkTest exte
- DEFAULT_COLLECTION.length() -1);
final int frozeUnique = unique;
Callable call = new Callable() {
+ @Override
public Object call() {
HttpSolrServer server;
try {
server = new HttpSolrServer(baseUrl);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
Create createCmd = new Create();
createCmd.setCoreName(collection);
createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
@@ -1507,7 +1306,7 @@ public class BasicDistributedZkTest exte
try {
// setup the server...
HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
- s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
+ s.setSoTimeout(30000);
s.setDefaultMaxConnectionsPerHost(100);
s.setMaxTotalConnections(100);
return s;
@@ -1517,39 +1316,42 @@ public class BasicDistributedZkTest exte
}
}
- volatile CloudSolrServer solrj;
-
- @Override
- protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
-
- if (r.nextBoolean())
- return super.queryServer(params);
-
- // use the distributed solrj client
- if (solrj == null) {
+ volatile CloudSolrServer commondCloudSolrServer;
+ private CloudSolrServer getCommonCloudSolrServer() {
+ if (commondCloudSolrServer == null) {
synchronized(this) {
try {
- CloudSolrServer server = new CloudSolrServer(zkServer.getZkAddress());
- server.setDefaultCollection(DEFAULT_COLLECTION);
- solrj = server;
+ commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
+ commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
+ commondCloudSolrServer.getLbServer().setConnectionTimeout(15000);
+ commondCloudSolrServer.getLbServer().setSoTimeout(30000);
+ commondCloudSolrServer.connect();
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
}
}
+ return commondCloudSolrServer;
+ }
+
+ @Override
+ protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
+
+ if (r.nextBoolean())
+ return super.queryServer(params);
if (r.nextBoolean())
params.set("collection",DEFAULT_COLLECTION);
- QueryResponse rsp = solrj.query(params);
+ QueryResponse rsp = getCommonCloudSolrServer().query(params);
return rsp;
}
@Override
public void tearDown() throws Exception {
super.tearDown();
- if (solrj != null) {
- solrj.shutdown();
+ if (commondCloudSolrServer != null) {
+ commondCloudSolrServer.shutdown();
}
System.clearProperty("numShards");
System.clearProperty("zkHost");
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Fri Jan 18 18:30:54 2013
@@ -21,8 +21,10 @@ import java.net.ConnectException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.http.client.HttpClient;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.Slow;
+
+import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
@@ -34,12 +36,11 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Slow
-@Ignore("ignore while investigating jenkins fails")
+@BadApple
public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {
public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class);
@@ -60,6 +61,7 @@ public class ChaosMonkeyNothingIsSafeTes
// can help to hide this when testing and looking at logs
//ignoreException("shard update error");
System.setProperty("numShards", Integer.toString(sliceCount));
+ useFactory("solr.StandardDirectoryFactory");
}
@Override
@@ -72,8 +74,8 @@ public class ChaosMonkeyNothingIsSafeTes
public ChaosMonkeyNothingIsSafeTest() {
super();
- sliceCount = 1;
- shardCount = 7;
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "2"));
+ shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "7"));
}
@Override
@@ -86,7 +88,7 @@ public class ChaosMonkeyNothingIsSafeTes
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
// make sure we have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
} // make sure we again have leaders for each shard
waitForRecoveriesToFinish(false);
@@ -140,12 +142,13 @@ public class ChaosMonkeyNothingIsSafeTes
indexThread.join();
}
- // we expect full throttle fails, but not cloud client...
- for (StopableThread indexThread : threads) {
- if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
- //assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
- }
- }
+ // we expect full throttle fails, but cloud client should not easily fail
+ // but it's allowed to fail and sometimes does, so commented out for now
+// for (StopableThread indexThread : threads) {
+// if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
+// assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
+// }
+// }
// try and wait for any replications and what not to finish...
@@ -156,7 +159,7 @@ public class ChaosMonkeyNothingIsSafeTes
// make sure we again have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
}
commit();
@@ -204,9 +207,12 @@ public class ChaosMonkeyNothingIsSafeTes
setName("FullThrottleStopableIndexingThread");
setDaemon(true);
this.clients = clients;
+ HttpClientUtil.setConnectionTimeout(httpClient, 15000);
+ HttpClientUtil.setSoTimeout(httpClient, 15000);
suss = new ConcurrentUpdateSolrServer(
((HttpSolrServer) clients.get(0)).getBaseURL(), httpClient, 8,
2) {
+ @Override
public void handleError(Throwable ex) {
log.warn("suss error", ex);
}
@@ -275,6 +281,7 @@ public class ChaosMonkeyNothingIsSafeTes
suss = new ConcurrentUpdateSolrServer(
((HttpSolrServer) clients.get(clientIndex)).getBaseURL(),
httpClient, 30, 3) {
+ @Override
public void handleError(Throwable ex) {
log.warn("suss error", ex);
}
@@ -282,12 +289,14 @@ public class ChaosMonkeyNothingIsSafeTes
}
}
+ @Override
public void safeStop() {
stop = true;
suss.shutdownNow();
httpClient.getConnectionManager().shutdown();
}
+ @Override
public int getFails() {
return fails.get();
}
@@ -296,6 +305,7 @@ public class ChaosMonkeyNothingIsSafeTes
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = getDoc(fields);
indexDoc(doc);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Fri Jan 18 18:30:54 2013
@@ -20,6 +20,8 @@ package org.apache.solr.cloud;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.SolrCore;
@@ -29,12 +31,13 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
-@Ignore("SOLR-3126")
+@Slow
+@BadApple
public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
private static final int BASE_RUN_LENGTH = 120000;
+ private static final int RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", Integer.toString(BASE_RUN_LENGTH)));
@BeforeClass
public static void beforeSuperClass() {
@@ -49,6 +52,8 @@ public class ChaosMonkeySafeLeaderTest e
@Before
@Override
public void setUp() throws Exception {
+ useFactory("solr.StandardDirectoryFactory");
+
super.setUp();
System.setProperty("numShards", Integer.toString(sliceCount));
@@ -64,8 +69,8 @@ public class ChaosMonkeySafeLeaderTest e
public ChaosMonkeySafeLeaderTest() {
super();
- sliceCount = 3;//atLeast(2);
- shardCount = 12;//atLeast(sliceCount*2);
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "3"));
+ shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "12"));
}
@Override
@@ -83,13 +88,13 @@ public class ChaosMonkeySafeLeaderTest e
List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
int threadCount = 2;
for (int i = 0; i < threadCount; i++) {
- StopableIndexingThread indexThread = new StopableIndexingThread(i * 50000, true);
+ StopableIndexingThread indexThread = new StopableIndexingThread(10000 + i*50000, true);
threads.add(indexThread);
indexThread.start();
}
chaosMonkey.startTheMonkey(false, 500);
- int runLength = atLeast(BASE_RUN_LENGTH);
+ int runLength = RUN_LENGTH;
Thread.sleep(runLength);
chaosMonkey.stopTheMonkey();
@@ -108,8 +113,10 @@ public class ChaosMonkeySafeLeaderTest e
}
// try and wait for any replications and what not to finish...
-
- waitForThingsToLevelOut(Integer.MAX_VALUE);//Math.round((runLength / 1000.0f / 3.0f)));
+
+ Thread.sleep(2000);
+
+ waitForThingsToLevelOut(180000);
checkShardConsistency(true, true);
@@ -135,6 +142,7 @@ public class ChaosMonkeySafeLeaderTest e
}
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java Fri Jan 18 18:30:54 2013
@@ -24,6 +24,9 @@ import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -32,7 +35,7 @@ import org.junit.Test;
public class ClusterStateTest extends SolrTestCaseJ4 {
@Test
public void testStoreAndRead() throws Exception {
- Map<String,Map<String,Slice>> collectionStates = new HashMap<String,Map<String,Slice>>();
+ Map<String,DocCollection> collectionStates = new HashMap<String,DocCollection>();
Set<String> liveNodes = new HashSet<String>();
liveNodes.add("node1");
liveNodes.add("node2");
@@ -49,12 +52,12 @@ public class ClusterStateTest extends So
slices.put("shard1", slice);
Slice slice2 = new Slice("shard2", sliceToProps, null);
slices.put("shard2", slice2);
- collectionStates.put("collection1", slices);
- collectionStates.put("collection2", slices);
+ collectionStates.put("collection1", new DocCollection("collection1", slices, null, DocRouter.DEFAULT));
+ collectionStates.put("collection2", new DocCollection("collection2", slices, null, DocRouter.DEFAULT));
ClusterState clusterState = new ClusterState(liveNodes, collectionStates);
byte[] bytes = ZkStateReader.toJSON(clusterState);
-
+ // System.out.println("#################### " + new String(bytes));
ClusterState loadedClusterState = ClusterState.load(null, bytes, liveNodes);
assertEquals("Provided liveNodes not used properly", 2, loadedClusterState
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java Fri Jan 18 18:30:54 2013
@@ -171,7 +171,7 @@ public class ClusterStateUpdateTest exte
Map<String,Slice> slices = null;
for (int i = 75; i > 0; i--) {
clusterState2 = zkController2.getClusterState();
- slices = clusterState2.getSlices("testcore");
+ slices = clusterState2.getSlicesMap("testcore");
if (slices != null && slices.containsKey("shard1")
&& slices.get("shard1").getReplicasMap().size() > 0) {
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Fri Jan 18 18:30:54 2013
@@ -132,7 +132,7 @@ public class FullSolrCloudDistribCmdsTes
private void testThatCantForwardToLeaderFails() throws Exception {
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
- ZkNodeProps props = zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard1");
+ ZkNodeProps props = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard1");
chaosMonkey.stopShard("shard1");
@@ -238,7 +238,8 @@ public class FullSolrCloudDistribCmdsTes
private void testIndexingWithSuss() throws Exception {
ConcurrentUpdateSolrServer suss = new ConcurrentUpdateSolrServer(
((HttpSolrServer) clients.get(0)).getBaseURL(), 3, 1);
-
+ suss.setConnectionTimeout(15000);
+ suss.setSoTimeout(30000);
for (int i=100; i<150; i++) {
index_specific(suss, id, i);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java Fri Jan 18 18:30:54 2013
@@ -256,7 +256,7 @@ public class LeaderElectionIntegrationTe
private String getLeader() throws InterruptedException {
- ZkNodeProps props = reader.getLeaderProps("collection1", "shard1", 30000);
+ ZkNodeProps props = reader.getLeaderRetry("collection1", "shard1", 30000);
String leader = props.getStr(ZkStateReader.NODE_NAME_PROP);
return leader;
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Fri Jan 18 18:30:54 2013
@@ -35,6 +35,7 @@ import javax.xml.parsers.ParserConfigura
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -81,9 +82,9 @@ public class OverseerTest extends SolrTe
private void deleteNode(final String path) {
try {
- Stat stat = zkClient.exists(path, null, false);
+ Stat stat = zkClient.exists(path, null, true);
if (stat != null) {
- zkClient.delete(path, stat.getVersion(), false);
+ zkClient.delete(path, stat.getVersion(), true);
}
} catch (KeeperException e) {
fail("Unexpected KeeperException!" + e);
@@ -148,7 +149,7 @@ public class OverseerTest extends SolrTe
}
private String getShardId(final String coreName) {
- Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(
+ Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(
collection);
if (slices != null) {
for (Slice slice : slices.values()) {
@@ -301,11 +302,14 @@ public class OverseerTest extends SolrTe
cloudStateSliceCount = 0;
reader.updateClusterState(true);
ClusterState state = reader.getClusterState();
- Map<String,Slice> slices = state.getSlices("collection1");
- for (String name : slices.keySet()) {
- cloudStateSliceCount += slices.get(name).getReplicasMap().size();
+ Map<String,Slice> slices = state.getSlicesMap("collection1");
+ if (slices != null) {
+ for (String name : slices.keySet()) {
+ cloudStateSliceCount += slices.get(name).getReplicasMap().size();
+ }
+ if (coreCount == cloudStateSliceCount) break;
}
- if (coreCount == cloudStateSliceCount) break;
+
Thread.sleep(200);
}
assertEquals("Unable to verify all cores have been assigned an id in cloudstate",
@@ -313,7 +317,7 @@ public class OverseerTest extends SolrTe
// make sure all cores have been returned an id
int assignedCount = 0;
- for (int i = 0; i < 90; i++) {
+ for (int i = 0; i < 120; i++) {
assignedCount = 0;
for (int j = 0; j < coreCount; j++) {
if (ids[j] != null) {
@@ -712,8 +716,8 @@ public class OverseerTest extends SolrTe
ClusterState state = reader.getClusterState();
int numFound = 0;
- for (Map<String,Slice> collection : state.getCollectionStates().values()) {
- for (Slice slice : collection.values()) {
+ for (DocCollection collection : state.getCollectionStates().values()) {
+ for (Slice slice : collection.getSlices()) {
if (slice.getReplicasMap().get("node1_core1") != null) {
numFound++;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Fri Jan 18 18:30:54 2013
@@ -82,11 +82,21 @@ public class RecoveryZkTest extends Abst
indexThread.join();
indexThread2.join();
- commit();
+ Thread.sleep(500);
+
+ waitForThingsToLevelOut(30);
+
+ Thread.sleep(1000);
+
+ waitForThingsToLevelOut(30);
+
+ Thread.sleep(1000);
+
+ waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
// test that leader and replica have same doc count
- checkShardConsistency("shard1", false);
+ checkShardConsistency("shard1", false, false);
SolrQuery query = new SolrQuery("*:*");
query.setParam("distrib", "false");
long client1Docs = shardToJetty.get("shard1").get(0).client.solrClient.query(query).getResults().getNumFound();
@@ -99,6 +109,7 @@ public class RecoveryZkTest extends Abst
//query("q", "*:*", "sort", "id desc");
}
+ @Override
protected void indexDoc(SolrInputDocument doc) throws IOException,
SolrServerException {
controlClient.add(doc);
@@ -124,6 +135,7 @@ public class RecoveryZkTest extends Abst
}
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);
Modified: lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Fri Jan 18 18:30:54 2013
@@ -33,7 +33,6 @@ import org.apache.solr.client.solrj.requ
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.servlet.SolrDispatchFilter;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -115,7 +114,7 @@ public class SyncSliceTest extends Abstr
waitForRecoveriesToFinish(false);
// shard should be inconsistent
- String shardFailMessage = checkShardConsistency("shard1", true);
+ String shardFailMessage = checkShardConsistency("shard1", true, false);
assertNotNull(shardFailMessage);
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -130,6 +129,8 @@ public class SyncSliceTest extends Abstr
baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
+ baseServer.setConnectionTimeout(15000);
+ baseServer.setSoTimeout(30000);
baseServer.request(request);
waitForThingsToLevelOut(15);
@@ -157,12 +158,11 @@ public class SyncSliceTest extends Abstr
chaosMonkey.killJetty(leaderJetty);
- // we are careful to make sure the downed node is no longer in the state,
- // because on some systems (especially freebsd w/ blackhole enabled), trying
- // to talk to a downed node causes grief
- waitToSeeDownInClusterState(leaderJetty, jetties);
-
- waitForThingsToLevelOut(45);
+ Thread.sleep(2000);
+
+ waitForThingsToLevelOut(90);
+
+ Thread.sleep(1000);
checkShardConsistency(false, true);
@@ -193,7 +193,7 @@ public class SyncSliceTest extends Abstr
//System.out.println("leader:" + leaderJetty.url);
//System.out.println("skip list:" + skipServers);
- // we are skipping one nodes
+ // we are skipping 2 nodes
assertEquals(2, skipServers.size());
// more docs than can peer sync
@@ -204,12 +204,16 @@ public class SyncSliceTest extends Abstr
commit();
+ Thread.sleep(1000);
+
waitForRecoveriesToFinish(false);
// shard should be inconsistent
- shardFailMessage = checkShardConsistency("shard1", true);
- assertNotNull(shardFailMessage);
+ shardFailMessage = waitTillInconsistent();
+ assertNotNull(
+ "shard1 should have just been set up to be inconsistent - but it's still consistent",
+ shardFailMessage);
jetties = new HashSet<CloudJettyRunner>();
jetties.addAll(shardToJetty.get("shard1"));
@@ -220,9 +224,11 @@ public class SyncSliceTest extends Abstr
// kill the current leader
chaosMonkey.killJetty(leaderJetty);
- waitToSeeDownInClusterState(leaderJetty, jetties);
+ Thread.sleep(3000);
+
+ waitForThingsToLevelOut(90);
- Thread.sleep(4000);
+ Thread.sleep(2000);
waitForRecoveriesToFinish(false);
@@ -230,6 +236,32 @@ public class SyncSliceTest extends Abstr
}
+ private String waitTillInconsistent() throws Exception, InterruptedException {
+ String shardFailMessage = null;
+
+ shardFailMessage = pollConsistency(shardFailMessage, 0);
+ shardFailMessage = pollConsistency(shardFailMessage, 3000);
+ shardFailMessage = pollConsistency(shardFailMessage, 5000);
+ shardFailMessage = pollConsistency(shardFailMessage, 8000);
+
+ return shardFailMessage;
+ }
+
+ private String pollConsistency(String shardFailMessage, int sleep)
+ throws InterruptedException, Exception {
+ try {
+ commit();
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
+ if (shardFailMessage == null) {
+ // try again
+ Thread.sleep(sleep);
+ shardFailMessage = checkShardConsistency("shard1", true, false);
+ }
+ return shardFailMessage;
+ }
+
private List<String> getRandomJetty() {
return getRandomOtherJetty(null, null);
}
@@ -251,17 +283,6 @@ public class SyncSliceTest extends Abstr
skipServers.add(cjetty.url + "/");
return skipServers;
}
-
- private void waitToSeeDownInClusterState(CloudJettyRunner leaderJetty,
- Set<CloudJettyRunner> jetties) throws InterruptedException {
-
- for (CloudJettyRunner cjetty : jetties) {
- waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
- .getFilter()).getCores().getZkController().getZkStateReader(),
- leaderJetty);
- }
- waitToSeeNotLive(cloudClient.getZkStateReader(), leaderJetty);
- }
protected void indexDoc(List<String> skipServers, Object... fields) throws IOException,
SolrServerException {
@@ -283,6 +304,7 @@ public class SyncSliceTest extends Abstr
}
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);