You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/01/08 04:40:37 UTC
svn commit: r1430130 [23/27] - in /lucene/dev/branches/lucene4547: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/solr/
dev-tools/maven/solr/contrib/analysis-extras/
dev-tools/maven/solr/contrib/cluste...
Modified: lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test-files/solr/solr.xml Tue Jan 8 03:40:16 2013
@@ -29,7 +29,15 @@
If 'null' (or absent), cores will not be manageable via request handler
-->
<cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}"
- hostContext="solr" zkClientTimeout="8000" numShards="${numShards:3}" shareSchema="${shareSchema:false}">
+ hostContext="${hostContext:solr}" zkClientTimeout="${solr.zkclienttimeout:30000}" numShards="${numShards:3}" shareSchema="${shareSchema:false}"
+ distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:60000}">
<core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"/>
+ <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
+ <int name="socketTimeout">${socketTimeout:60000}</int>
+ <int name="connTimeout">${connTimeout:15000}</int>
+ </shardHandlerFactory>
</cores>
+
+
+
</solr>
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java Tue Jan 8 03:40:16 2013
@@ -30,9 +30,11 @@ import org.apache.solr.client.solrj.requ
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.SolrCore;
import org.junit.BeforeClass;
+import org.junit.AfterClass;
public class AnalysisAfterCoreReloadTest extends SolrTestCaseJ4 {
+ private static String tmpSolrHome;
int port = 0;
static final String context = "/solr";
@@ -40,9 +42,17 @@ public class AnalysisAfterCoreReloadTest
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml", "schema.xml");
+ createTempDir();
+ tmpSolrHome = TEMP_DIR + File.separator + AnalysisAfterCoreReloadTest.class.getSimpleName() + System.currentTimeMillis();
+ FileUtils.copyDirectory(new File(TEST_HOME()), new File(tmpSolrHome).getAbsoluteFile());
+ initCore("solrconfig.xml", "schema.xml", new File(tmpSolrHome).getAbsolutePath());
}
+ @AfterClass
+ public static void AfterClass() throws Exception {
+ FileUtils.deleteDirectory(new File(tmpSolrHome).getAbsoluteFile());
+ }
+
public void testStopwordsAfterCoreReload() throws Exception {
SolrInputDocument doc = new SolrInputDocument();
doc.setField( "id", "42" );
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java Tue Jan 8 03:40:16 2013
@@ -21,20 +21,22 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.util.HashMap;
-import java.util.Map;
-import java.util.List;
import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.StoredDocument;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext.Context;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.RateLimitedDirectoryWrapper;
import org.apache.lucene.util.English;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
@@ -55,8 +57,6 @@ import org.apache.solr.search.DocIterato
import org.apache.solr.search.DocList;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.util.RefCounted;
-
-
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -126,15 +126,28 @@ public class BasicFunctionalityTest exte
clearIndex();
// test merge factor picked up
+ // and for rate limited settings
SolrCore core = h.getCore();
- RefCounted<IndexWriter> iw = ((DirectUpdateHandler2) core
+ RefCounted<IndexWriter> iwr = ((DirectUpdateHandler2) core
.getUpdateHandler()).getSolrCoreState().getIndexWriter(core);
try {
- assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy) iw
- .get().getConfig().getMergePolicy()).getMergeFactor());
+ IndexWriter iw = iwr.get();
+ assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy) iw.getConfig().getMergePolicy()).getMergeFactor());
+
+ Directory dir = iw.getDirectory();
+
+ if (dir instanceof MockDirectoryWrapper) {
+ dir = ((MockDirectoryWrapper)dir).getDelegate();
+ }
+
+ assertTrue(dir.getClass().getName(), dir instanceof RateLimitedDirectoryWrapper);
+ assertEquals(Double.valueOf(1000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.DEFAULT));
+ assertEquals(Double.valueOf(2000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.FLUSH));
+ assertEquals(Double.valueOf(3000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.MERGE));
+ assertEquals(Double.valueOf(4000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.READ));
} finally {
- iw.decref();
+ iwr.decref();
}
// test stats call
NamedList stats = core.getStatistics();
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/OutputWriterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/OutputWriterTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/OutputWriterTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/OutputWriterTest.java Tue Jan 8 03:40:16 2013
@@ -109,13 +109,16 @@ public class OutputWriterTest extends So
public UselessOutputWriter() {}
+ @Override
public void init(NamedList n) {}
+ @Override
public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response)
throws IOException {
writer.write(USELESS_OUTPUT);
}
+ @Override
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
return CONTENT_TYPE_TEXT_UTF8;
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java Tue Jan 8 03:40:16 2013
@@ -43,6 +43,7 @@ public class TestDistributedGrouping ext
String tdate_b = "b_n_tdt";
String oddField="oddField_s";
+ @Override
public void doTest() throws Exception {
del("*:*");
commit();
@@ -50,6 +51,8 @@ public class TestDistributedGrouping ext
handle.clear();
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
+ handle.put("grouped", UNORDERED); // distrib grouping doesn't guarantee order of top level group commands
+
// Test distributed grouping with empty indices
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " asc, id asc");
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " asc, id asc", "hl","true","hl.fl",t1);
@@ -162,6 +165,30 @@ public class TestDistributedGrouping ext
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc");
query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc");
+ // SOLR-4150: what if group.query has no matches,
+ // or only matches on one shard
+ query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "group.query", "id:5", // single doc, so only one shard will have it
+ "group.limit", 10, "sort", i1 + " asc, id asc");
+ handle.put(t1 + ":this_will_never_match", SKIP); // :TODO: SOLR-4181
+ query("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "group.query", t1 + ":this_will_never_match",
+ "group.limit", 10, "sort", i1 + " asc, id asc");
+
+ // SOLR-4164: main query matches nothing, or only matches on one shard
+ query("q", "bogus_s:nothing", // no docs match
+ "group", "true",
+ "group.query", t1 + ":this_will_never_match",
+ "group.field", i1,
+ "fl", "id", "group.limit", "2", "group.format", "simple");
+ query("q", "id:5", // one doc matches, so only one shard
+ "rows", 100, "fl", "id," + i1, "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "group.field", i1,
+ "group.limit", 10, "sort", i1 + " asc, id asc");
+
// SOLR-3109
query("q", t1 + ":eggs", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", tlong + " asc, id asc");
query("q", i1 + ":232", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", tlong + " asc, id asc");
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestGroupingSearch.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestGroupingSearch.java Tue Jan 8 03:40:16 2013
@@ -502,6 +502,18 @@ public class TestGroupingSearch extends
"'doclist':{'numFound':4,'start':0,'docs':[{'id':'3'},{'id':'4'},{'id':'2'}]}}}"
);
+ // group.query that matches nothing
+ assertJQ(req("fq",filt,
+ "q","{!func}"+f2,
+ "group","true",
+ "group.query","id:[2 TO 5]",
+ "group.query","id:1000",
+ "fl","id",
+ "group.limit","3")
+ ,"/grouped/id:[2 TO 5]=={'matches':10,'doclist':{'numFound':4,'start':0,'docs':[{'id':'3'},{'id':'4'},{'id':'2'}]}}"
+ ,"/grouped/id:1000=={'matches':10,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ );
+
// group.query and offset
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.query","id:[2 TO 5]", "fl","id", "group.limit","3", "group.offset","2")
,"/grouped=={'id:[2 TO 5]':{'matches':10," +
@@ -573,6 +585,34 @@ public class TestGroupingSearch extends
, "/grouped/foo_i=={'matches':10,'doclist':"
+ "{'numFound':10,'start':1,'docs':[{'id':'10'},{'id':'3'},{'id':'6'}]}}"
);
+
+ //////////////////////// grouping where main query matches nothing
+ assertJQ(req("fq", filt, "q", "bogus_s:nothing", "group", "true", "group.field", f, "fl", "id", "group.limit", "2", "group.format", "simple")
+ , "/grouped/foo_i=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ );
+ assertJQ(req("fq",filt, "q","bogus_s:nothing", "group","true",
+ "group.query","id:[2 TO 5]",
+ "group.query","id:[5 TO 5]",
+ "group.field",f,
+ "rows","1",
+ "fl","id", "group.limit","2")
+ ,"/grouped/id:[2 TO 5]=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ ,"/grouped/id:[5 TO 5]=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ ,"/grouped/"+f+"=={'matches':0,'groups':[]}"
+ );
+ assertJQ(req("fq",filt,
+ "q","bogus_s:nothing",
+ "group","true",
+ "group.query","id:[2 TO 5]",
+ "group.query","id:1000",
+ "fl","id",
+ "group.limit","3")
+ ,"/grouped/id:[2 TO 5]=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ ,"/grouped/id:1000=={'matches':0,'doclist':{'numFound':0,'start':0,'docs':[]}}"
+ );
+
+
+
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java Tue Jan 8 03:40:16 2013
@@ -16,7 +16,6 @@
*/
package org.apache.solr;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.util.AbstractSolrTestCase;
@@ -37,9 +36,6 @@ import java.util.Properties;
*
* @since solr 1.4
*/
-@ThreadLeakFilters(defaultFilters = true, filters = {
- SolrIgnoredThreadsFilter.class
-})
public class TestSolrCoreProperties extends LuceneTestCase {
private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
JettySolrRunner solrJetty;
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestTrie.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestTrie.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/TestTrie.java Tue Jan 8 03:40:16 2013
@@ -16,6 +16,8 @@
*/
package org.apache.solr;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.DateField;
import org.apache.solr.schema.FieldType;
@@ -25,6 +27,7 @@ import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.TimeZone;
@@ -47,6 +50,38 @@ public class TestTrie extends SolrTestCa
clearIndex();
super.tearDown();
}
+
+ @Test
+ public void testTokenizer() throws Exception {
+ FieldType type = h.getCore().getSchema().getFieldType("tint");
+ assertTrue(type instanceof TrieField);
+
+ String value = String.valueOf(random().nextInt());
+ TokenStream ts = type.getAnalyzer().tokenStream("dummy", new StringReader(value));
+ OffsetAttribute ofsAtt = ts.addAttribute(OffsetAttribute.class);
+ ts.reset();
+ int count = 0;
+ while (ts.incrementToken()) {
+ count++;
+ assertEquals(0, ofsAtt.startOffset());
+ assertEquals(value.length(), ofsAtt.endOffset());
+ }
+ final int precStep = ((TrieField) type).getPrecisionStep();
+ assertEquals( (32 + precStep - 1) / precStep, count);
+ ts.end();
+ assertEquals(value.length(), ofsAtt.startOffset());
+ assertEquals(value.length(), ofsAtt.endOffset());
+ ts.close();
+
+ // Test empty one:
+ ts = type.getAnalyzer().tokenStream("dummy", new StringReader(""));
+ ts.reset();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ assertEquals(0, ofsAtt.startOffset());
+ assertEquals(0, ofsAtt.endOffset());
+ ts.close();
+ }
@Test
public void testTrieIntRangeSearch() throws Exception {
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Tue Jan 8 03:40:16 2013
@@ -17,13 +17,24 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
@@ -32,7 +43,11 @@ import org.apache.solr.common.cloud.ZkNo
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.ReplicationHandler;
import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.BeforeClass;
/**
@@ -110,7 +125,7 @@ public class BasicDistributedZk2Test ext
// TODO: bring this to it's own method?
// try indexing to a leader that has no replicas up
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
- ZkNodeProps leaderProps = zkStateReader.getLeaderProps(
+ ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(
DEFAULT_COLLECTION, SHARD2);
String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP);
@@ -200,26 +215,6 @@ public class BasicDistributedZk2Test ext
jetties.addAll(shardToJetty.get(SHARD2));
jetties.remove(deadShard);
- // wait till live nodes drops by 1
- int liveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
- int tries = 50;
- while(oldLiveNodes == liveNodes) {
- Thread.sleep(100);
- if (tries-- == 0) {
- fail("We expected a node to drop...");
- }
- liveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
- }
- assertEquals(4, liveNodes);
-
- int cnt = 0;
- for (CloudJettyRunner cjetty : jetties) {
- waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
- .getFilter()).getCores().getZkController().getZkStateReader(),
- deadShard, cnt++);
- }
- waitToSeeNotLive(cloudClient.getZkStateReader(), deadShard);
-
// ensure shard is dead
try {
index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1,
@@ -269,7 +264,14 @@ public class BasicDistributedZk2Test ext
UpdateRequest ureq = new UpdateRequest();
ureq.add(doc);
// ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
- ureq.process(cloudClient);
+
+ try {
+ ureq.process(cloudClient);
+ } catch(SolrServerException e){
+ // try again
+ Thread.sleep(500);
+ ureq.process(cloudClient);
+ }
commit();
@@ -319,14 +321,7 @@ public class BasicDistributedZk2Test ext
// recover over 100 docs so we do more than just peer sync (replicate recovery)
chaosMonkey.stopJetty(deadShard);
-
- for (CloudJettyRunner cjetty : jetties) {
- waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
- .getFilter()).getCores().getZkController().getZkStateReader(),
- deadShard);
- }
- waitToSeeNotLive(cloudClient.getZkStateReader(), deadShard);
-
+
for (int i = 0; i < 226; i++) {
doc = new SolrInputDocument();
doc.addField("id", 2000 + i);
@@ -345,9 +340,110 @@ public class BasicDistributedZk2Test ext
// make sure we have published we are recovering
Thread.sleep(1500);
+ waitForThingsToLevelOut(15);
+
+ Thread.sleep(500);
+
waitForRecoveriesToFinish(false);
checkShardConsistency(true, false);
+
+ // try a backup command
+ final HttpSolrServer client = (HttpSolrServer) shardToJetty.get(SHARD2).get(0).client.solrClient;
+ System.out.println("base url: "+ client.getBaseURL());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("qt", "/replication");
+ params.set("command", "backup");
+
+ QueryRequest request = new QueryRequest(params);
+ NamedList<Object> results = client.request(request );
+ System.out.println("results:" + results);
+
+
+ checkForBackupSuccess(client);
+
+ }
+ private void checkForBackupSuccess(final HttpSolrServer client)
+ throws InterruptedException, IOException {
+ class CheckStatus extends Thread {
+ volatile String fail = null;
+ volatile String response = null;
+ volatile boolean success = false;
+ final Pattern p = Pattern
+ .compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
+
+ CheckStatus() {}
+
+ @Override
+ public void run() {
+ String masterUrl = client.getBaseURL() + "/replication?command="
+ + ReplicationHandler.CMD_DETAILS;
+ URL url;
+ InputStream stream = null;
+ try {
+ url = new URL(masterUrl);
+ stream = url.openStream();
+ response = IOUtils.toString(stream, "UTF-8");
+ if (response.contains("<str name=\"status\">success</str>")) {
+ Matcher m = p.matcher(response);
+ if (!m.find()) {
+ fail("could not find the completed timestamp in response.");
+ }
+
+ success = true;
+ }
+ stream.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail = e.getMessage();
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+
+ };
+ }
+ ;
+ SolrCore core = ((SolrDispatchFilter) shardToJetty.get(SHARD2).get(0).jetty
+ .getDispatchFilter().getFilter()).getCores().getCore("collection1");
+ String ddir;
+ try {
+ ddir = core.getDataDir();
+ } finally {
+ core.close();
+ }
+ File dataDir = new File(ddir);
+
+ int waitCnt = 0;
+ CheckStatus checkStatus = new CheckStatus();
+ while (true) {
+ checkStatus.run();
+ if (checkStatus.fail != null) {
+ fail(checkStatus.fail);
+ }
+ if (checkStatus.success) {
+ break;
+ }
+ Thread.sleep(200);
+ if (waitCnt == 20) {
+ fail("Backup success not detected:" + checkStatus.response);
+ }
+ waitCnt++;
+ }
+
+ File[] files = dataDir.listFiles(new FilenameFilter() {
+
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name.startsWith("snapshot")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(Arrays.asList(files).toString(), 1, files.length);
+ File snapDir = files[0];
+
+ AbstractSolrTestCase.recurseDelete(snapDir); // clean up the snap dir
}
private void addNewReplica() throws Exception {
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Tue Jan 8 03:40:16 2013
@@ -23,10 +23,8 @@ import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -38,7 +36,6 @@ import java.util.concurrent.ThreadPoolEx
import java.util.concurrent.TimeUnit;
import org.apache.lucene.util.LuceneTestCase.Slow;
-import org.apache.lucene.util._TestUtil;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
@@ -48,16 +45,13 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
-import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload;
import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -165,7 +159,7 @@ public class BasicDistributedZkTest exte
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
// make sure we have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
} // make sure we again have leaders for each shard
waitForRecoveriesToFinish(false);
@@ -340,7 +334,6 @@ public class BasicDistributedZkTest exte
testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit");
testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit");
- testCollectionsAPI();
testCoreUnloadAndLeaders();
testUnloadLotsOfCores();
testStopAndStartCoresInOneInstance();
@@ -366,7 +359,8 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(0);
String url1 = getBaseUrl(client);
HttpSolrServer server = new HttpSolrServer(url1);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
server.request(createCmd);
createCmd = new Create();
@@ -428,6 +422,8 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(0);
String url1 = getBaseUrl(client);
HttpSolrServer server = new HttpSolrServer(url1);
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
Create createCmd = new Create();
createCmd.setCoreName("unloadcollection1");
@@ -530,7 +526,7 @@ public class BasicDistributedZkTest exte
}
// ensure there is a leader
- zkStateReader.getLeaderProps("unloadcollection", "shard1", 15000);
+ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
addClient = new HttpSolrServer(url2 + "/unloadcollection2");
// add a few docs while the leader is down
@@ -572,7 +568,7 @@ public class BasicDistributedZkTest exte
}
}
- zkStateReader.getLeaderProps("unloadcollection", "shard1", 15000);
+ zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
// set this back
@@ -618,13 +614,14 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(2);
String url3 = getBaseUrl(client);
final HttpSolrServer server = new HttpSolrServer(url3);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(60000);
ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
- int cnt = atLeast(6);
+ int cnt = random().nextInt(12) + 1;
- // create the 6 cores
+ // create the cores
createCores(server, executor, "multiunload", 2, cnt);
executor.shutdown();
@@ -658,7 +655,8 @@ public class BasicDistributedZkTest exte
SolrServer client = clients.get(0);
String url3 = getBaseUrl(client);
final HttpSolrServer server = new HttpSolrServer(url3);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
@@ -677,7 +675,7 @@ public class BasicDistributedZkTest exte
ChaosMonkey.start(cloudJettys.get(0).jetty);
cloudClient.getZkStateReader().updateClusterState(true);
try {
- cloudClient.getZkStateReader().getLeaderProps("multiunload2", "shard1", 30000);
+ cloudClient.getZkStateReader().getLeaderRetry("multiunload2", "shard1", 30000);
} catch (SolrException e) {
printLayout();
throw e;
@@ -715,7 +713,6 @@ public class BasicDistributedZkTest exte
}
}
-
private String getBaseUrl(SolrServer client) {
String url2 = ((HttpSolrServer) client).getBaseURL()
.substring(
@@ -725,270 +722,36 @@ public class BasicDistributedZkTest exte
return url2;
}
-
- private void testCollectionsAPI() throws Exception {
-
- // TODO: fragile - because we dont pass collection.confName, it will only
- // find a default if a conf set with a name matching the collection name is found, or
- // if there is only one conf set. That and the fact that other tests run first in this
- // env make this pretty fragile
-
- // create new collections rapid fire
- Map<String,List<Integer>> collectionInfos = new HashMap<String,List<Integer>>();
- int cnt = atLeast(3);
-
- for (int i = 0; i < cnt; i++) {
- int numShards = _TestUtil.nextInt(random(), 0, shardCount) + 1;
- int numReplica = _TestUtil.nextInt(random(), 0, 3) + 1;
- int maxShardsPerNode = (((numShards * (numReplica + 1)) / getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size())) + 1;
- createCollection(collectionInfos, i, numShards, numReplica, maxShardsPerNode);
- }
-
- Set<Entry<String,List<Integer>>> collectionInfosEntrySet = collectionInfos.entrySet();
- for (Entry<String,List<Integer>> entry : collectionInfosEntrySet) {
- String collection = entry.getKey();
- List<Integer> list = entry.getValue();
- checkForCollection(collection, list);
-
- String url = getUrlFromZk(collection);
-
- HttpSolrServer collectionClient = new HttpSolrServer(url);
-
- // poll for a second - it can take a moment before we are ready to serve
- waitForNon403or404or503(collectionClient);
- }
- ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
- for (int j = 0; j < cnt; j++) {
- waitForRecoveriesToFinish("awholynewcollection_" + j, zkStateReader, false);
- }
-
- List<String> collectionNameList = new ArrayList<String>();
- collectionNameList.addAll(collectionInfos.keySet());
- String collectionName = collectionNameList.get(random().nextInt(collectionNameList.size()));
-
- String url = getUrlFromZk(collectionName);
-
- HttpSolrServer collectionClient = new HttpSolrServer(url);
-
-
- // lets try and use the solrj client to index a couple documents
- SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1,
- "humpty dumpy sat on a wall");
- SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1,
- "humpty dumpy3 sat on a walls");
- SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1,
- "humpty dumpy2 sat on a walled");
-
- collectionClient.add(doc1);
-
- collectionClient.add(doc2);
-
- collectionClient.add(doc3);
-
- collectionClient.commit();
-
- assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
-
- // lets try a collection reload
-
- // get core open times
- Map<String,Long> urlToTimeBefore = new HashMap<String,Long>();
- collectStartTimes(collectionName, urlToTimeBefore);
- assertTrue(urlToTimeBefore.size() > 0);
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.RELOAD.toString());
- params.set("name", collectionName);
- QueryRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- // we can use this client because we just want base url
- final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(0)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
-
- createNewSolrServer("", baseUrl).request(request);
-
- // reloads make take a short while
- boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore);
- assertTrue("some core start times did not change on reload", allTimesAreCorrect);
-
-
- waitForRecoveriesToFinish("awholynewcollection_" + (cnt - 1), zkStateReader, false);
-
- // remove a collection
- params = new ModifiableSolrParams();
- params.set("action", CollectionAction.DELETE.toString());
- params.set("name", collectionName);
- request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- createNewSolrServer("", baseUrl).request(request);
-
- // ensure its out of the state
- checkForMissingCollection(collectionName);
-
- //collectionNameList.remove(collectionName);
-
- // remove an unknown collection
- params = new ModifiableSolrParams();
- params.set("action", CollectionAction.DELETE.toString());
- params.set("name", "unknown_collection");
- request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- createNewSolrServer("", baseUrl).request(request);
-
- // create another collection should still work
- params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
-
- params.set("numShards", 1);
- params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, 1);
- collectionName = "acollectionafterbaddelete";
-
- params.set("name", collectionName);
- request = new QueryRequest(params);
- request.setPath("/admin/collections");
- createNewSolrServer("", baseUrl).request(request);
-
- List<Integer> list = new ArrayList<Integer> (2);
- list.add(1);
- list.add(1);
- checkForCollection(collectionName, list);
-
- url = getUrlFromZk(collectionName);
-
- collectionClient = new HttpSolrServer(url);
-
- // poll for a second - it can take a moment before we are ready to serve
- waitForNon403or404or503(collectionClient);
-
- for (int j = 0; j < cnt; j++) {
- waitForRecoveriesToFinish(collectionName, zkStateReader, false);
- }
-
- // test maxShardsPerNode
- int liveNodes = getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size();
- int numShards = (liveNodes/2) + 1;
- int numReplica = 1;
- int maxShardsPerNode = 1;
- collectionInfos = new HashMap<String,List<Integer>>();
- createCollection(collectionInfos, cnt, numShards, numReplica, maxShardsPerNode);
-
- // TODO: enable this check after removing the 60 second wait in it
- //checkCollectionIsNotCreated(collectionInfos.keySet().iterator().next());
- }
-
-
protected void createCollection(Map<String,List<Integer>> collectionInfos,
- int i, int numShards, int numReplica, int maxShardsPerNode) throws SolrServerException, IOException {
+ String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
- params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplica);
+ params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
- String collectionName = "awholynewcollection_" + i;
+ if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
+
int clientIndex = random().nextInt(2);
List<Integer> list = new ArrayList<Integer>();
list.add(numShards);
- list.add(numReplica);
- list.add(maxShardsPerNode);
+ list.add(numReplicas);
collectionInfos.put(collectionName, list);
params.set("name", collectionName);
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
- final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
-
- createNewSolrServer("", baseUrl).request(request);
- }
-
- private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
-
-
- long timeoutAt = System.currentTimeMillis() + 30000;
-
- boolean allTimesAreCorrect = false;
- while (System.currentTimeMillis() < timeoutAt) {
- Map<String,Long> urlToTimeAfter = new HashMap<String,Long>();
- collectStartTimes(collectionName, urlToTimeAfter);
+ if (client == null) {
+ final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
+ 0,
+ ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
+ - DEFAULT_COLLECTION.length() - 1);
- boolean retry = false;
- Set<Entry<String,Long>> entries = urlToTimeBefore.entrySet();
- for (Entry<String,Long> entry : entries) {
- Long beforeTime = entry.getValue();
- Long afterTime = urlToTimeAfter.get(entry.getKey());
- assertNotNull(afterTime);
- if (afterTime <= beforeTime) {
- retry = true;
- break;
- }
-
- }
- if (!retry) {
- allTimesAreCorrect = true;
- break;
- }
- }
- return allTimesAreCorrect;
- }
-
- private void collectStartTimes(String collectionName,
- Map<String,Long> urlToTime) throws SolrServerException, IOException {
- Map<String,DocCollection> collections = getCommonCloudSolrServer().getZkStateReader()
- .getClusterState().getCollectionStates();
- if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
-
- Iterator<Entry<String,Slice>> it = slices.entrySet().iterator();
- while (it.hasNext()) {
- Entry<String,Slice> sliceEntry = it.next();
- Map<String,Replica> sliceShards = sliceEntry.getValue().getReplicasMap();
- Iterator<Entry<String,Replica>> shardIt = sliceShards.entrySet()
- .iterator();
- while (shardIt.hasNext()) {
- Entry<String,Replica> shardEntry = shardIt.next();
- ZkCoreNodeProps coreProps = new ZkCoreNodeProps(shardEntry.getValue());
- CoreAdminResponse mcr = CoreAdminRequest.getStatus(
- coreProps.getCoreName(),
- new HttpSolrServer(coreProps.getBaseUrl()));
- long before = mcr.getStartTime(coreProps.getCoreName()).getTime();
- urlToTime.put(coreProps.getCoreUrl(), before);
- }
- }
+ createNewSolrServer("", baseUrl).request(request);
} else {
- throw new IllegalArgumentException("Could not find collection in :"
- + collections.keySet());
+ client.request(request);
}
}
-
- private String getUrlFromZk(String collection) {
- ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
- Map<String,Slice> slices = clusterState.getCollectionStates().get(collection).getSlicesMap();
-
- if (slices == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
- }
-
- for (Map.Entry<String,Slice> entry : slices.entrySet()) {
- Slice slice = entry.getValue();
- Map<String,Replica> shards = slice.getReplicasMap();
- Set<Map.Entry<String,Replica>> shardEntries = shards.entrySet();
- for (Map.Entry<String,Replica> shardEntry : shardEntries) {
- final ZkNodeProps node = shardEntry.getValue();
- if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
- return ZkCoreNodeProps.getCoreUrl(node.getStr(ZkStateReader.BASE_URL_PROP), collection); //new ZkCoreNodeProps(node).getCoreUrl();
- }
- }
- }
-
- throw new RuntimeException("Could not find a live node for collection:" + collection);
- }
private ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
@@ -999,38 +762,12 @@ public class BasicDistributedZkTest exte
return new ZkCoreNodeProps(leader);
}
- private void waitForNon403or404or503(HttpSolrServer collectionClient)
- throws Exception {
- SolrException exp = null;
- long timeoutAt = System.currentTimeMillis() + 30000;
-
- while (System.currentTimeMillis() < timeoutAt) {
- boolean missing = false;
-
- try {
- collectionClient.query(new SolrQuery("*:*"));
- } catch (SolrException e) {
- if (!(e.code() == 403 || e.code() == 503 || e.code() == 404)) {
- throw e;
- }
- exp = e;
- missing = true;
- }
- if (!missing) {
- return;
- }
- Thread.sleep(50);
- }
-
- fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
- }
-
- private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList) {
+ private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
int expectedSlices = numShardsNumReplicaList.get(0);
// The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
- int expectedShardsPerSlice = numShardsNumReplicaList.get(1) + 1;
+ int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
Map<String,DocCollection> collections = clusterState
@@ -1043,6 +780,11 @@ public class BasicDistributedZkTest exte
}
int totalShards = 0;
for (String sliceName : slices.keySet()) {
+ for (Replica replica : slices.get(sliceName).getReplicas()) {
+ if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
+ return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
+ }
+ }
totalShards += slices.get(sliceName).getReplicas().size();
}
if (totalShards != expectedTotalShards) {
@@ -1054,14 +796,14 @@ public class BasicDistributedZkTest exte
}
}
- private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList)
+ private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
throws Exception {
// check for an expectedSlices new collection - we poll the state
long timeoutAt = System.currentTimeMillis() + 120000;
boolean success = false;
String checkResult = "Didnt get to perform a single check";
while (System.currentTimeMillis() < timeoutAt) {
- checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList);
+ checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
if (checkResult == null) {
success = true;
break;
@@ -1076,15 +818,15 @@ public class BasicDistributedZkTest exte
private void checkCollectionIsNotCreated(String collectionName)
throws Exception {
- // TODO: this method not called because of below sleep
- Thread.sleep(60000);
+ // TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
+ Thread.sleep(10000);
assertFalse(collectionName + " not supposed to exist", getCommonCloudSolrServer().getZkStateReader().getClusterState().getCollections().contains(collectionName));
}
private void checkForMissingCollection(String collectionName)
throws Exception {
// check for a collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 15000;
+ long timeoutAt = System.currentTimeMillis() + 30000;
boolean found = true;
while (System.currentTimeMillis() < timeoutAt) {
getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
@@ -1210,6 +952,8 @@ public class BasicDistributedZkTest exte
private Long getNumCommits(HttpSolrServer solrServer) throws
SolrServerException, IOException {
HttpSolrServer server = new HttpSolrServer(solrServer.getBaseURL());
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/admin/mbeans?key=updateHandler&stats=true");
// use generic request to avoid extra processing of queries
@@ -1397,11 +1141,13 @@ public class BasicDistributedZkTest exte
List<SolrServer> collectionClients, final String baseUrl, final int num,
final String shardId) {
Callable call = new Callable() {
+ @Override
public Object call() {
HttpSolrServer server;
try {
server = new HttpSolrServer(baseUrl);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
Create createCmd = new Create();
createCmd.setRoles("none");
createCmd.setCoreName(collection + num);
@@ -1524,11 +1270,13 @@ public class BasicDistributedZkTest exte
- DEFAULT_COLLECTION.length() -1);
final int frozeUnique = unique;
Callable call = new Callable() {
+ @Override
public Object call() {
HttpSolrServer server;
try {
server = new HttpSolrServer(baseUrl);
-
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
Create createCmd = new Create();
createCmd.setCoreName(collection);
createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
@@ -1558,7 +1306,7 @@ public class BasicDistributedZkTest exte
try {
// setup the server...
HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
- s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
+ s.setSoTimeout(30000);
s.setDefaultMaxConnectionsPerHost(100);
s.setMaxTotalConnections(100);
return s;
@@ -1575,6 +1323,8 @@ public class BasicDistributedZkTest exte
try {
commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
+ commondCloudSolrServer.getLbServer().setConnectionTimeout(15000);
+ commondCloudSolrServer.getLbServer().setSoTimeout(30000);
commondCloudSolrServer.connect();
} catch (MalformedURLException e) {
throw new RuntimeException(e);
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Tue Jan 8 03:40:16 2013
@@ -21,8 +21,10 @@ import java.net.ConnectException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.http.client.HttpClient;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.Slow;
+
+import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
@@ -34,12 +36,11 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Slow
-@Ignore("ignore while investigating jenkins fails")
+@BadApple
public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {
public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class);
@@ -60,6 +61,7 @@ public class ChaosMonkeyNothingIsSafeTes
// can help to hide this when testing and looking at logs
//ignoreException("shard update error");
System.setProperty("numShards", Integer.toString(sliceCount));
+ useFactory("solr.StandardDirectoryFactory");
}
@Override
@@ -72,8 +74,8 @@ public class ChaosMonkeyNothingIsSafeTes
public ChaosMonkeyNothingIsSafeTest() {
super();
- sliceCount = 1;
- shardCount = 7;
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "2"));
+ shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "7"));
}
@Override
@@ -86,7 +88,7 @@ public class ChaosMonkeyNothingIsSafeTes
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
// make sure we have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
} // make sure we again have leaders for each shard
waitForRecoveriesToFinish(false);
@@ -140,12 +142,13 @@ public class ChaosMonkeyNothingIsSafeTes
indexThread.join();
}
- // we expect full throttle fails, but not cloud client...
- for (StopableThread indexThread : threads) {
- if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
- //assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
- }
- }
+ // we expect full throttle fails, but cloud client should not easily fail
+ // but it's allowed to fail and sometimes does, so commented out for now
+// for (StopableThread indexThread : threads) {
+// if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
+// assertEquals(0, ((StopableIndexingThread) indexThread).getFails());
+// }
+// }
// try and wait for any replications and what not to finish...
@@ -156,7 +159,7 @@ public class ChaosMonkeyNothingIsSafeTes
// make sure we again have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
- zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+ zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
}
commit();
@@ -204,9 +207,12 @@ public class ChaosMonkeyNothingIsSafeTes
setName("FullThrottleStopableIndexingThread");
setDaemon(true);
this.clients = clients;
+ HttpClientUtil.setConnectionTimeout(httpClient, 15000);
+ HttpClientUtil.setSoTimeout(httpClient, 15000);
suss = new ConcurrentUpdateSolrServer(
((HttpSolrServer) clients.get(0)).getBaseURL(), httpClient, 8,
2) {
+ @Override
public void handleError(Throwable ex) {
log.warn("suss error", ex);
}
@@ -275,6 +281,7 @@ public class ChaosMonkeyNothingIsSafeTes
suss = new ConcurrentUpdateSolrServer(
((HttpSolrServer) clients.get(clientIndex)).getBaseURL(),
httpClient, 30, 3) {
+ @Override
public void handleError(Throwable ex) {
log.warn("suss error", ex);
}
@@ -282,12 +289,14 @@ public class ChaosMonkeyNothingIsSafeTes
}
}
+ @Override
public void safeStop() {
stop = true;
suss.shutdownNow();
httpClient.getConnectionManager().shutdown();
}
+ @Override
public int getFails() {
return fails.get();
}
@@ -296,6 +305,7 @@ public class ChaosMonkeyNothingIsSafeTes
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = getDoc(fields);
indexDoc(doc);
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Tue Jan 8 03:40:16 2013
@@ -20,6 +20,8 @@ package org.apache.solr.cloud;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.SolrCore;
@@ -29,12 +31,13 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
-@Ignore("SOLR-3126")
+@Slow
+@BadApple
public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
private static final int BASE_RUN_LENGTH = 120000;
+ private static final int RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", Integer.toString(BASE_RUN_LENGTH)));
@BeforeClass
public static void beforeSuperClass() {
@@ -49,6 +52,8 @@ public class ChaosMonkeySafeLeaderTest e
@Before
@Override
public void setUp() throws Exception {
+ useFactory("solr.StandardDirectoryFactory");
+
super.setUp();
System.setProperty("numShards", Integer.toString(sliceCount));
@@ -64,8 +69,8 @@ public class ChaosMonkeySafeLeaderTest e
public ChaosMonkeySafeLeaderTest() {
super();
- sliceCount = 3;//atLeast(2);
- shardCount = 12;//atLeast(sliceCount*2);
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "3"));
+ shardCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "12"));
}
@Override
@@ -83,13 +88,13 @@ public class ChaosMonkeySafeLeaderTest e
List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
int threadCount = 2;
for (int i = 0; i < threadCount; i++) {
- StopableIndexingThread indexThread = new StopableIndexingThread(i * 50000, true);
+ StopableIndexingThread indexThread = new StopableIndexingThread(10000 + i*50000, true);
threads.add(indexThread);
indexThread.start();
}
chaosMonkey.startTheMonkey(false, 500);
- int runLength = atLeast(BASE_RUN_LENGTH);
+ int runLength = RUN_LENGTH;
Thread.sleep(runLength);
chaosMonkey.stopTheMonkey();
@@ -108,8 +113,10 @@ public class ChaosMonkeySafeLeaderTest e
}
// try and wait for any replications and what not to finish...
-
- waitForThingsToLevelOut(Integer.MAX_VALUE);//Math.round((runLength / 1000.0f / 3.0f)));
+
+ Thread.sleep(2000);
+
+ waitForThingsToLevelOut(180000);
checkShardConsistency(true, true);
@@ -135,6 +142,7 @@ public class ChaosMonkeySafeLeaderTest e
}
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Tue Jan 8 03:40:16 2013
@@ -132,7 +132,7 @@ public class FullSolrCloudDistribCmdsTes
private void testThatCantForwardToLeaderFails() throws Exception {
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
- ZkNodeProps props = zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard1");
+ ZkNodeProps props = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard1");
chaosMonkey.stopShard("shard1");
@@ -238,7 +238,8 @@ public class FullSolrCloudDistribCmdsTes
private void testIndexingWithSuss() throws Exception {
ConcurrentUpdateSolrServer suss = new ConcurrentUpdateSolrServer(
((HttpSolrServer) clients.get(0)).getBaseURL(), 3, 1);
-
+ suss.setConnectionTimeout(15000);
+ suss.setSoTimeout(30000);
for (int i=100; i<150; i++) {
index_specific(suss, id, i);
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java Tue Jan 8 03:40:16 2013
@@ -256,7 +256,7 @@ public class LeaderElectionIntegrationTe
private String getLeader() throws InterruptedException {
- ZkNodeProps props = reader.getLeaderProps("collection1", "shard1", 30000);
+ ZkNodeProps props = reader.getLeaderRetry("collection1", "shard1", 30000);
String leader = props.getStr(ZkStateReader.NODE_NAME_PROP);
return leader;
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Tue Jan 8 03:40:16 2013
@@ -82,9 +82,9 @@ public class OverseerTest extends SolrTe
private void deleteNode(final String path) {
try {
- Stat stat = zkClient.exists(path, null, false);
+ Stat stat = zkClient.exists(path, null, true);
if (stat != null) {
- zkClient.delete(path, stat.getVersion(), false);
+ zkClient.delete(path, stat.getVersion(), true);
}
} catch (KeeperException e) {
fail("Unexpected KeeperException!" + e);
@@ -303,10 +303,13 @@ public class OverseerTest extends SolrTe
reader.updateClusterState(true);
ClusterState state = reader.getClusterState();
Map<String,Slice> slices = state.getSlicesMap("collection1");
- for (String name : slices.keySet()) {
- cloudStateSliceCount += slices.get(name).getReplicasMap().size();
+ if (slices != null) {
+ for (String name : slices.keySet()) {
+ cloudStateSliceCount += slices.get(name).getReplicasMap().size();
+ }
+ if (coreCount == cloudStateSliceCount) break;
}
- if (coreCount == cloudStateSliceCount) break;
+
Thread.sleep(200);
}
assertEquals("Unable to verify all cores have been assigned an id in cloudstate",
@@ -314,7 +317,7 @@ public class OverseerTest extends SolrTe
// make sure all cores have been returned an id
int assignedCount = 0;
- for (int i = 0; i < 90; i++) {
+ for (int i = 0; i < 120; i++) {
assignedCount = 0;
for (int j = 0; j < coreCount; j++) {
if (ids[j] != null) {
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Tue Jan 8 03:40:16 2013
@@ -82,11 +82,21 @@ public class RecoveryZkTest extends Abst
indexThread.join();
indexThread2.join();
- commit();
+ Thread.sleep(500);
+
+ waitForThingsToLevelOut(30);
+
+ Thread.sleep(1000);
+
+ waitForThingsToLevelOut(30);
+
+ Thread.sleep(1000);
+
+ waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
// test that leader and replica have same doc count
- checkShardConsistency("shard1", false);
+ checkShardConsistency("shard1", false, false);
SolrQuery query = new SolrQuery("*:*");
query.setParam("distrib", "false");
long client1Docs = shardToJetty.get("shard1").get(0).client.solrClient.query(query).getResults().getNumFound();
@@ -99,6 +109,7 @@ public class RecoveryZkTest extends Abst
//query("q", "*:*", "sort", "id desc");
}
+ @Override
protected void indexDoc(SolrInputDocument doc) throws IOException,
SolrServerException {
controlClient.add(doc);
@@ -124,6 +135,7 @@ public class RecoveryZkTest extends Abst
}
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java Tue Jan 8 03:40:16 2013
@@ -41,10 +41,18 @@ import org.junit.Ignore;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class ShardRoutingTest extends AbstractFullDistribZkTestBase {
+
+ String bucket1 = "shard1"; // shard1: top bits:10 80000000:bfffffff
+ String bucket2 = "shard2"; // shard2: top bits:11 c0000000:ffffffff
+ String bucket3 = "shard3"; // shard3: top bits:00 00000000:3fffffff
+ String bucket4 = "shard4"; // shard4: top bits:01 40000000:7fffffff
+
+
@BeforeClass
public static void beforeShardHashingTest() throws Exception {
// TODO: we use an fs based dir because something
@@ -109,6 +117,8 @@ public class ShardRoutingTest extends Ab
waitForRecoveriesToFinish(false);
doHashingTest();
+ doTestNumRequests();
+ doAtomicUpdate();
testFinished = true;
} finally {
@@ -122,15 +132,12 @@ public class ShardRoutingTest extends Ab
private void doHashingTest() throws Exception {
+ log.info("### STARTING doHashingTest");
assertEquals(4, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlices().size());
String shardKeys = ShardParams.SHARD_KEYS;
// for now, we know how ranges will be distributed to shards.
// may have to look it up in clusterstate if that assumption changes.
- String bucket1 = "shard1"; // shard1: top bits:10 80000000:bfffffff
- String bucket2 = "shard2"; // shard2: top bits:11 c0000000:ffffffff
- String bucket3 = "shard3"; // shard3: top bits:00 00000000:3fffffff
- String bucket4 = "shard4"; // shard4: top bits:01 40000000:7fffffff
doAddDoc("b!doc1");
doAddDoc("c!doc2");
@@ -196,10 +203,93 @@ public class ShardRoutingTest extends Ab
doAddDoc("e!doc4");
commit();
+ }
+
+
+
+
+
+ public void doTestNumRequests() throws Exception {
+ log.info("### STARTING doTestNumRequests");
+ List<CloudJettyRunner> runners = shardToJetty.get(bucket1);
+ CloudJettyRunner leader = shardToLeaderJetty.get(bucket1);
+ CloudJettyRunner replica = null;
+ for (CloudJettyRunner r : runners) {
+ if (r != leader) replica = r;
+ }
+
+ long nStart = getNumRequests();
+ leader.client.solrClient.add( sdoc("id","b!doc1") );
+ long nEnd = getNumRequests();
+ assertEquals(2, nEnd - nStart); // one request to leader, which makes another to a replica
+
+
+ nStart = getNumRequests();
+ replica.client.solrClient.add( sdoc("id","b!doc1") );
+ nEnd = getNumRequests();
+ assertEquals(3, nEnd - nStart); // orig request + replica forwards to leader, which forward back to replica.
+
+ nStart = getNumRequests();
+ replica.client.solrClient.add( sdoc("id","b!doc1") );
+ nEnd = getNumRequests();
+ assertEquals(3, nEnd - nStart); // orig request + replica forwards to leader, which forward back to replica.
+
+ CloudJettyRunner leader2 = shardToLeaderJetty.get(bucket2);
+
+
+ nStart = getNumRequests();
+ replica.client.solrClient.query( params("q","*:*", "shards",bucket1) );
+ nEnd = getNumRequests();
+ assertEquals(1, nEnd - nStart); // short circuit should prevent distrib search
+
+ nStart = getNumRequests();
+ replica.client.solrClient.query( params("q","*:*", "shard.keys","b!") );
+ nEnd = getNumRequests();
+ assertEquals(1, nEnd - nStart); // short circuit should prevent distrib search
+
+ nStart = getNumRequests();
+ leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!") );
+ nEnd = getNumRequests();
+ assertEquals(3, nEnd - nStart); // original + 2 phase distrib search. we could improve this!
+
+ nStart = getNumRequests();
+ leader2.client.solrClient.query( params("q","*:*") );
+ nEnd = getNumRequests();
+ assertEquals(9, nEnd - nStart); // original + 2 phase distrib search * 4 shards.
+
+ nStart = getNumRequests();
+ leader2.client.solrClient.query( params("q","*:*", "shard.keys","b!,d!") );
+ nEnd = getNumRequests();
+ assertEquals(5, nEnd - nStart); // original + 2 phase distrib search * 2 shards.
+ }
+
+ public void doAtomicUpdate() throws Exception {
+ log.info("### STARTING doAtomicUpdate");
+ int nClients = clients.size();
+ assertEquals(8, nClients);
+
+ int expectedVal = 0;
+ for (SolrServer client : clients) {
+ client.add(sdoc("id", "b!doc", "foo_i", map("inc",1)));
+ expectedVal++;
+
+ QueryResponse rsp = client.query(params("qt","/get", "id","b!doc"));
+ Object val = ((Map)rsp.getResponse().get("doc")).get("foo_i");
+ assertEquals((Integer)expectedVal, val);
+ }
}
+ long getNumRequests() {
+ long n = controlJetty.getDebugFilter().getTotalRequests();
+ for (JettySolrRunner jetty : jettys) {
+ n += jetty.getDebugFilter().getTotalRequests();
+ }
+ return n;
+ }
+
+
void doAddDoc(String id) throws Exception {
index("id",id);
// todo - target diff servers and use cloud clients as well as non-cloud clients
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Tue Jan 8 03:40:16 2013
@@ -33,7 +33,6 @@ import org.apache.solr.client.solrj.requ
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.servlet.SolrDispatchFilter;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -115,7 +114,7 @@ public class SyncSliceTest extends Abstr
waitForRecoveriesToFinish(false);
// shard should be inconsistent
- String shardFailMessage = checkShardConsistency("shard1", true);
+ String shardFailMessage = checkShardConsistency("shard1", true, false);
assertNotNull(shardFailMessage);
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -130,6 +129,8 @@ public class SyncSliceTest extends Abstr
baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
+ baseServer.setConnectionTimeout(15000);
+ baseServer.setSoTimeout(30000);
baseServer.request(request);
waitForThingsToLevelOut(15);
@@ -157,12 +158,11 @@ public class SyncSliceTest extends Abstr
chaosMonkey.killJetty(leaderJetty);
- // we are careful to make sure the downed node is no longer in the state,
- // because on some systems (especially freebsd w/ blackhole enabled), trying
- // to talk to a downed node causes grief
- waitToSeeDownInClusterState(leaderJetty, jetties);
-
- waitForThingsToLevelOut(45);
+ Thread.sleep(2000);
+
+ waitForThingsToLevelOut(90);
+
+ Thread.sleep(1000);
checkShardConsistency(false, true);
@@ -193,7 +193,7 @@ public class SyncSliceTest extends Abstr
//System.out.println("leader:" + leaderJetty.url);
//System.out.println("skip list:" + skipServers);
- // we are skipping one nodes
+ // we are skipping 2 nodes
assertEquals(2, skipServers.size());
// more docs than can peer sync
@@ -204,12 +204,16 @@ public class SyncSliceTest extends Abstr
commit();
+ Thread.sleep(1000);
+
waitForRecoveriesToFinish(false);
// shard should be inconsistent
- shardFailMessage = checkShardConsistency("shard1", true);
- assertNotNull(shardFailMessage);
+ shardFailMessage = waitTillInconsistent();
+ assertNotNull(
+ "shard1 should have just been set up to be inconsistent - but it's still consistent",
+ shardFailMessage);
jetties = new HashSet<CloudJettyRunner>();
jetties.addAll(shardToJetty.get("shard1"));
@@ -220,9 +224,11 @@ public class SyncSliceTest extends Abstr
// kill the current leader
chaosMonkey.killJetty(leaderJetty);
- waitToSeeDownInClusterState(leaderJetty, jetties);
+ Thread.sleep(3000);
+
+ waitForThingsToLevelOut(90);
- Thread.sleep(4000);
+ Thread.sleep(2000);
waitForRecoveriesToFinish(false);
@@ -230,6 +236,32 @@ public class SyncSliceTest extends Abstr
}
+ private String waitTillInconsistent() throws Exception, InterruptedException {
+ String shardFailMessage = null;
+
+ shardFailMessage = pollConsistency(shardFailMessage, 0);
+ shardFailMessage = pollConsistency(shardFailMessage, 3000);
+ shardFailMessage = pollConsistency(shardFailMessage, 5000);
+ shardFailMessage = pollConsistency(shardFailMessage, 8000);
+
+ return shardFailMessage;
+ }
+
+ private String pollConsistency(String shardFailMessage, int sleep)
+ throws InterruptedException, Exception {
+ try {
+ commit();
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
+ if (shardFailMessage == null) {
+ // try again
+ Thread.sleep(sleep);
+ shardFailMessage = checkShardConsistency("shard1", true, false);
+ }
+ return shardFailMessage;
+ }
+
private List<String> getRandomJetty() {
return getRandomOtherJetty(null, null);
}
@@ -251,17 +283,6 @@ public class SyncSliceTest extends Abstr
skipServers.add(cjetty.url + "/");
return skipServers;
}
-
- private void waitToSeeDownInClusterState(CloudJettyRunner leaderJetty,
- Set<CloudJettyRunner> jetties) throws InterruptedException {
-
- for (CloudJettyRunner cjetty : jetties) {
- waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
- .getFilter()).getCores().getZkController().getZkStateReader(),
- leaderJetty);
- }
- waitToSeeNotLive(cloudClient.getZkStateReader(), leaderJetty);
- }
protected void indexDoc(List<String> skipServers, Object... fields) throws IOException,
SolrServerException {
@@ -283,6 +304,7 @@ public class SyncSliceTest extends Abstr
}
// skip the randoms - they can deadlock...
+ @Override
protected void indexr(Object... fields) throws Exception {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java Tue Jan 8 03:40:16 2013
@@ -47,13 +47,22 @@ public class TestHashPartitioner extends
assertEquals(0x00000000, ranges.get(1).min);
assertEquals(0x7fffffff, ranges.get(1).max);
- ranges = hp.partitionRange(2, 0, 0x7fffffff);
+ ranges = hp.partitionRange(2, new DocRouter.Range(0, 0x7fffffff));
assertEquals(0x00000000, ranges.get(0).min);
assertEquals(0x3fffffff, ranges.get(0).max);
assertEquals(0x40000000, ranges.get(1).min);
assertEquals(0x7fffffff, ranges.get(1).max);
- for (int i = 1; i <= 30000; i += 13) {
+ int defaultLowerBits = 0x0000ffff;
+
+ for (int i = 1; i <= 30000; i++) {
+ // start skipping at higher numbers
+ if (i > 100) i+=13;
+ else if (i > 1000) i+=31;
+ else if (i > 5000) i+=101;
+
+ long rangeSize = 0x0000000100000000L / i;
+
ranges = hp.partitionRange(i, hp.fullRange());
assertEquals(i, ranges.size());
assertTrue("First range does not start before " + Integer.MIN_VALUE
@@ -69,6 +78,32 @@ public class TestHashPartitioner extends
assertEquals(range, newRange);
}
+ // ensure that ranges are contiguous and that size deviations are not too large.
+ int lastEnd = Integer.MIN_VALUE - 1;
+ for (Range range : ranges) {
+ int currStart = range.min;
+ int currEnd = range.max;
+ assertEquals(lastEnd+1, currStart);
+
+ if (ranges.size() < 4000) {
+ // ranges should be rounded to avoid crossing hash domains
+ assertEquals(defaultLowerBits, currEnd & defaultLowerBits);
+
+ // given our rounding condition that domains should be less than 1/16 of the step size,
+ // this means that any sizing deviations should also be less than 1/16th of the idealized range size.
+ // boolean round = rangeStep >= (1<<bits)*16;
+
+ long currRangeSize = (long)currEnd - (long)currStart;
+ long error = Math.abs(rangeSize - currRangeSize);
+ assertTrue( error < rangeSize/16);
+ }
+
+
+ // String s = range.toString();
+ // Range newRange = hp.fromString(s);
+ // assertEquals(range, newRange);
+ lastEnd = currEnd;
+ }
}
}
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java Tue Jan 8 03:40:16 2013
@@ -94,11 +94,23 @@ public class ZkCLITest extends SolrTestC
assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/core0", true));
assertTrue(zkClient.exists(ZkController.CONFIGS_ZKNODE + "/core1", true));
-
-
}
@Test
+ public void testBootstrapWithChroot() throws Exception {
+ String chroot = "/foo/bar";
+ assertFalse(zkClient.exists(chroot, true));
+
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress() + chroot,
+ "-cmd", "bootstrap", "-solrhome", ExternalPaths.EXAMPLE_HOME};
+
+ ZkCLI.main(args);
+
+ assertTrue(zkClient.exists(chroot + ZkController.CONFIGS_ZKNODE
+ + "/collection1", true));
+ }
+
+ @Test
public void testMakePath() throws Exception {
// test bootstrap_conf
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Tue Jan 8 03:40:16 2013
@@ -31,7 +31,6 @@ import org.apache.solr.core.CoreContaine
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.util.ExternalPaths;
import org.apache.zookeeper.CreateMode;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,6 +48,88 @@ public class ZkControllerTest extends So
initCore();
}
+ public void testNodeNameUrlConversion() throws Exception {
+
+ // nodeName from parts
+ assertEquals("localhost:8888_solr",
+ ZkController.generateNodeName("localhost", "8888", "solr"));
+ assertEquals("localhost:8888_solr",
+ ZkController.generateNodeName("localhost", "8888", "/solr"));
+ assertEquals("localhost:8888_solr",
+ ZkController.generateNodeName("localhost", "8888", "/solr/"));
+ // root context
+ assertEquals("localhost:8888_",
+ ZkController.generateNodeName("localhost", "8888", ""));
+ assertEquals("localhost:8888_",
+ ZkController.generateNodeName("localhost", "8888", "/"));
+ // subdir
+ assertEquals("foo-bar:77_solr%2Fsub_dir",
+ ZkController.generateNodeName("foo-bar", "77", "solr/sub_dir"));
+ assertEquals("foo-bar:77_solr%2Fsub_dir",
+ ZkController.generateNodeName("foo-bar", "77", "/solr/sub_dir"));
+ assertEquals("foo-bar:77_solr%2Fsub_dir",
+ ZkController.generateNodeName("foo-bar", "77", "/solr/sub_dir/"));
+
+ // setup a SolrZkClient to do some getBaseUrlForNodeName testing
+ String zkDir = dataDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+
+ ZkTestServer server = new ZkTestServer(zkDir);
+ try {
+ server.run();
+
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+ AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+
+ SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+ try {
+
+ // getBaseUrlForNodeName
+ assertEquals("http://zzz.xxx:1234/solr",
+ zkClient.getBaseUrlForNodeName("zzz.xxx:1234_solr"));
+ assertEquals("http://xxx:99",
+ zkClient.getBaseUrlForNodeName("xxx:99_"));
+ assertEquals("http://foo-bar.baz.org:9999/some_dir",
+ zkClient.getBaseUrlForNodeName("foo-bar.baz.org:9999_some_dir"));
+ assertEquals("http://foo-bar.baz.org:9999/solr/sub_dir",
+ zkClient.getBaseUrlForNodeName("foo-bar.baz.org:9999_solr%2Fsub_dir"));
+
+ // generateNodeName + getBaseUrlForNodeName
+ assertEquals("http://foo:9876/solr",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo","9876","solr")));
+ assertEquals("http://foo:9876/solr",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo","9876","/solr")));
+ assertEquals("http://foo:9876/solr",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo","9876","/solr/")));
+ assertEquals("http://foo.bar.com:9876/solr/sub_dir",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo.bar.com","9876","solr/sub_dir")));
+ assertEquals("http://foo.bar.com:9876/solr/sub_dir",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo.bar.com","9876","/solr/sub_dir/")));
+ assertEquals("http://foo-bar:9876",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo-bar","9876","")));
+ assertEquals("http://foo-bar:9876",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo-bar","9876","/")));
+ assertEquals("http://foo-bar.com:80/some_dir",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo-bar.com","80","some_dir")));
+ assertEquals("http://foo-bar.com:80/some_dir",
+ zkClient.getBaseUrlForNodeName
+ (ZkController.generateNodeName("foo-bar.com","80","/some_dir")));
+ } finally {
+ zkClient.close();
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
@Test
public void testReadConfigName() throws Exception {
String zkDir = dataDir.getAbsolutePath() + File.separator
@@ -82,7 +163,7 @@ public class ZkControllerTest extends So
cc = getCoreContainer();
ZkController zkController = new ZkController(cc, server.getZkAddress(), TIMEOUT, 10000,
- "127.0.0.1", "8983", "solr", "0", new CurrentCoreDescriptorProvider() {
+ "127.0.0.1", "8983", "solr", "0", 10000, 10000, new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {
@@ -122,7 +203,7 @@ public class ZkControllerTest extends So
cc = getCoreContainer();
zkController = new ZkController(cc, server.getZkAddress(),
- TIMEOUT, 10000, "127.0.0.1", "8983", "solr", "0", new CurrentCoreDescriptorProvider() {
+ TIMEOUT, 10000, "127.0.0.1", "8983", "solr", "0", 10000, 10000, new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {
@@ -143,7 +224,7 @@ public class ZkControllerTest extends So
}
testFinished = true;
} finally {
- if (!testFinished) {
+ if (!testFinished & zkController != null) {
zkController.getZkClient().printLayoutToStdOut();
}
@@ -159,7 +240,13 @@ public class ZkControllerTest extends So
}
private CoreContainer getCoreContainer() {
- return new CoreContainer(TEMP_DIR.getAbsolutePath());
+ CoreContainer cc = new CoreContainer(TEMP_DIR.getAbsolutePath()) {
+ {
+ initShardHandler(null);
+ }
+ };
+
+ return cc;
}
@Override
Modified: lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java?rev=1430130&r1=1430129&r2=1430130&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java (original)
+++ lucene/dev/branches/lucene4547/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java Tue Jan 8 03:40:16 2013
@@ -209,6 +209,7 @@ public class ZkSolrClientTest extends Ab
zkClient.getChildren("/collections", new Watcher() {
+ @Override
public void process(WatchedEvent event) {
if (DEBUG) {
System.out.println("children changed");