You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/13 19:09:35 UTC
svn commit: r1231216 -
/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/
Author: markrmiller
Date: Fri Jan 13 18:09:34 2012
New Revision: 1231216
URL: http://svn.apache.org/viewvc?rev=1231216&view=rev
Log:
add new chaos monkey test, ignored for now
Added:
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (with props)
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (contents, props changed)
- copied, changed from r1231197, lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySolrCloudTest.java
Removed:
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySolrCloudTest.java
Modified:
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java?rev=1231216&r1=1231215&r2=1231216&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java Fri Jan 13 18:09:34 2012
@@ -246,7 +246,7 @@ public class BasicSolrCloudTest extends
commit();
- checkShardConsistency(false);
+ checkShardConsistency();
}
private void testOptimisticUpdate(QueryResponse results) throws Exception {
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java?rev=1231216&r1=1231215&r2=1231216&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java Fri Jan 13 18:09:34 2012
@@ -47,7 +47,6 @@ public class ChaosMonkey {
private static final int CONLOSS_PERCENT = 3; //30%
private static final int EXPIRE_PERCENT = 4; //40%
- private static final boolean DONTKILLLEADER = true;
private Map<String,List<CloudJettyRunner>> shardToJetty;
private ZkTestServer zkServer;
@@ -63,6 +62,7 @@ public class ChaosMonkey {
private Map<String,List<SolrServer>> shardToClient;
private boolean expireSessions;
private boolean causeConnectionLoss;
+ private boolean killLeaders;
public ChaosMonkey(ZkTestServer zkServer, ZkStateReader zkStateReader,
String collection, Map<String,List<CloudJettyRunner>> shardToJetty,
@@ -92,7 +92,7 @@ public class ChaosMonkey {
public void expireRandomSession() throws KeeperException, InterruptedException {
String sliceName = getRandomSlice();
- JettySolrRunner jetty = getRandomJetty(sliceName, DONTKILLLEADER);
+ JettySolrRunner jetty = getRandomJetty(sliceName, killLeaders);
if (jetty != null) {
expireSession(jetty);
expires.incrementAndGet();
@@ -102,7 +102,7 @@ public class ChaosMonkey {
public void randomConnectionLoss() throws KeeperException, InterruptedException {
String sliceName = getRandomSlice();
- JettySolrRunner jetty = getRandomJetty(sliceName, DONTKILLLEADER);
+ JettySolrRunner jetty = getRandomJetty(sliceName, killLeaders);
if (jetty != null) {
causeConnectionLoss(jetty);
connloss.incrementAndGet();
@@ -210,7 +210,7 @@ public class ChaosMonkey {
}
public JettySolrRunner stopRandomShard(String slice) throws Exception {
- JettySolrRunner jetty = getRandomJetty(slice, DONTKILLLEADER);
+ JettySolrRunner jetty = getRandomJetty(slice, killLeaders);
if (jetty != null) {
stopJetty(jetty);
}
@@ -235,14 +235,14 @@ public class ChaosMonkey {
}
public JettySolrRunner killRandomShard(String slice) throws Exception {
- JettySolrRunner jetty = getRandomJetty(slice, DONTKILLLEADER);
+ JettySolrRunner jetty = getRandomJetty(slice, killLeaders);
if (jetty != null) {
killJetty(jetty);
}
return jetty;
}
- public JettySolrRunner getRandomJetty(String slice, boolean dontkillleader) throws KeeperException, InterruptedException {
+ public JettySolrRunner getRandomJetty(String slice, boolean killLeader) throws KeeperException, InterruptedException {
// get latest cloud state
zkStateReader.updateCloudState(true);
Slice theShards = zkStateReader.getCloudState().getSlices(collection)
@@ -298,7 +298,7 @@ public class ChaosMonkey {
ZkNodeProps leader = zkStateReader.getLeaderProps(collection, slice);
- if (dontkillleader && leader.get(ZkStateReader.NODE_NAME_PROP).equals(jetties.get(index).nodeName)) {
+ if (!killLeader && leader.get(ZkStateReader.NODE_NAME_PROP).equals(jetties.get(index).nodeName)) {
// we don't kill leaders...
return null;
}
@@ -320,7 +320,8 @@ public class ChaosMonkey {
// synchronously starts and stops shards randomly, unless there is only one
// active shard up for a slice or if there is one active and others recovering
- public void startTheMonkey() {
+ public void startTheMonkey(boolean killLeaders) {
+ this.killLeaders = killLeaders;
stop = false;
new Thread() {
private List<JettySolrRunner> deadPool = new ArrayList<JettySolrRunner>();
Added: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1231216&view=auto
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (added)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Fri Jan 13 18:09:34 2012
@@ -0,0 +1,164 @@
+package org.apache.solr.cloud;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+@Ignore
+public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest {
+
+ @BeforeClass
+ public static void beforeSuperClass() throws Exception {
+
+ }
+
+ @AfterClass
+ public static void afterSuperClass() throws Exception {
+
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ // we expect this time of exception as shards go up and down...
+ ignoreException("shard update error ");
+ ignoreException("Connection refused");
+ ignoreException("interrupted waiting for shard update response");
+ ignoreException("org\\.mortbay\\.jetty\\.EofException");
+ ignoreException("java\\.lang\\.InterruptedException");
+ ignoreException("java\\.nio\\.channels\\.ClosedByInterruptException");
+
+
+ // sometimes we cannot get the same port
+ ignoreException("java\\.net\\.BindException: Address already in use");
+
+ System.setProperty("numShards", Integer.toString(sliceCount));
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ super.tearDown();
+ resetExceptionIgnores();
+ }
+
+ public ChaosMonkeyNothingIsSafeTest() {
+ super();
+ shardCount = atLeast(6);
+ sliceCount = atLeast(2);
+ }
+
+ @Override
+ public void doTest() throws Exception {
+
+ handle.clear();
+ handle.put("QTime", SKIPVAL);
+ handle.put("timestamp", SKIPVAL);
+
+ // we cannot do delete by query
+ // as it's not supported for recovery
+ //del("*:*");
+
+ List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
+ int threadCount = atLeast(2);
+ for (int i = 0; i < threadCount; i++) {
+ StopableIndexingThread indexThread = new StopableIndexingThread(i * 50000, true);
+ threads.add(indexThread);
+ indexThread.start();
+ }
+
+ chaosMonkey.startTheMonkey(true);
+
+ Thread.sleep(atLeast(8000));
+
+ chaosMonkey.stopTheMonkey();
+
+ for (StopableIndexingThread indexThread : threads) {
+ indexThread.safeStop();
+ }
+
+ // wait for stop...
+ for (StopableIndexingThread indexThread : threads) {
+ indexThread.join();
+ }
+
+
+ // fails will happen...
+// for (StopableIndexingThread indexThread : threads) {
+// assertEquals(0, indexThread.getFails());
+// }
+
+ // try and wait for any replications and what not to finish...
+
+ // wait until there are no recoveries...
+ waitForThingsToLevelOut();
+
+ checkShardConsistency(false, false);
+
+ if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
+ }
+
+ private void waitForThingsToLevelOut() throws KeeperException,
+ InterruptedException, Exception, IOException, URISyntaxException {
+ int cnt = 0;
+ boolean retry = false;
+ do {
+ waitForRecoveriesToFinish(VERBOSE);
+
+ commit();
+
+ updateMappingsFromZk(jettys, clients);
+
+ Set<String> theShards = shardToClient.keySet();
+ String failMessage = null;
+ for (String shard : theShards) {
+ failMessage = checkShardConsistency(shard, false);
+ }
+
+ if (failMessage != null) {
+ retry = true;
+ }
+ cnt++;
+ if (cnt > 10) break;
+ Thread.sleep(4000);
+ } while (retry);
+ }
+
+ // skip the randoms - they can deadlock...
+ protected void indexr(Object... fields) throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, fields);
+ addFields(doc, "rnd_b", true);
+ indexDoc(doc);
+ }
+
+}
Copied: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (from r1231197, lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySolrCloudTest.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?p2=lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java&p1=lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySolrCloudTest.java&r1=1231197&r2=1231216&rev=1231216&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySolrCloudTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Fri Jan 13 18:09:34 2012
@@ -31,7 +31,7 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-public class ChaosMonkeySolrCloudTest extends FullSolrCloudTest {
+public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest {
@BeforeClass
public static void beforeSuperClass() throws Exception {
@@ -69,7 +69,7 @@ public class ChaosMonkeySolrCloudTest ex
resetExceptionIgnores();
}
- public ChaosMonkeySolrCloudTest() {
+ public ChaosMonkeySafeLeaderTest() {
super();
shardCount = atLeast(6);
sliceCount = atLeast(2);
@@ -94,7 +94,7 @@ public class ChaosMonkeySolrCloudTest ex
indexThread.start();
}
- chaosMonkey.startTheMonkey();
+ chaosMonkey.startTheMonkey(false);
Thread.sleep(atLeast(8000));
@@ -118,7 +118,7 @@ public class ChaosMonkeySolrCloudTest ex
// wait until there are no recoveries...
waitForThingsToLevelOut();
- checkShardConsistency(false);
+ checkShardConsistency();
if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1231216&r1=1231215&r2=1231216&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Fri Jan 13 18:09:34 2012
@@ -861,10 +861,10 @@ public class FullSolrCloudTest extends A
}
protected void checkShardConsistency() throws Exception {
- checkShardConsistency(false);
+ checkShardConsistency(true, false);
}
- protected void checkShardConsistency(boolean verbose) throws Exception {
+ protected void checkShardConsistency(boolean checkVsControl, boolean verbose) throws Exception {
long docs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
if (verbose) System.out.println("Control Docs:" + docs);