You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/08/08 20:15:58 UTC
svn commit: r1370869 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/test/org/apache/solr/cloud/ solr/solrj/
solr/solrj/src/java/org/apache/solr/client/solrj/impl/
solr/solrj/src/test-files/solrj/solr/ solr/solrj/src/test-files/s...
Author: markrmiller
Date: Wed Aug 8 18:15:57 2012
New Revision: 1370869
URL: http://svn.apache.org/viewvc?rev=1370869&view=rev
Log:
still favor leaders with CloudSolrServer, but also fall back to replicas after leaders
Added:
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
- copied unchanged from r1370864, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/collection1/
- copied from r1370864, lucene/dev/trunk/solr/solrj/src/test-files/solrj/solr/collection1/
lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/solr.xml
- copied unchanged from r1370864, lucene/dev/trunk/solr/solrj/src/test-files/solrj/solr/solr.xml
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
- copied unchanged from r1370864, lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/
- copied from r1370864, lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/
Removed:
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/AbstractZkTestCase.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkTestServer.java
lucene/dev/branches/branch_4x/solr/solrj/src/test-files/solrj/solr/conf/
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Wed Aug 8 18:15:57 2012
@@ -71,10 +71,11 @@ import org.apache.solr.update.SolrCmdDis
import org.apache.solr.util.DefaultSolrThreadFactory;
/**
- *
+ * This test simply does a bunch of basic things in solrcloud mode and asserts things
+ * work as expected.
*/
@Slow
-public class BasicDistributedZkTest extends AbstractDistributedZkTestCase {
+public class BasicDistributedZkTest extends AbstractDistribZkTestBase {
private static final String DEFAULT_COLLECTION = "collection1";
private static final boolean DEBUG = false;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Wed Aug 8 18:15:57 2012
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
@Slow
@Ignore("ignore while investigating jenkins fails")
-public class ChaosMonkeyNothingIsSafeTest extends FullSolrCloudTest {
+public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase {
public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class);
private static final int BASE_RUN_LENGTH = 180000;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Wed Aug 8 18:15:57 2012
@@ -32,7 +32,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
@Ignore("SOLR-3126")
-public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest {
+public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
private static final int BASE_RUN_LENGTH = 120000;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Wed Aug 8 18:15:57 2012
@@ -23,8 +23,8 @@ import org.apache.lucene.util.LuceneTest
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
@@ -37,14 +37,13 @@ import org.apache.solr.common.params.Mod
import org.apache.solr.update.VersionInfo;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
/**
* Super basic testing, no shard restarting or anything.
*/
@Slow
-public class FullSolrCloudDistribCmdsTest extends FullSolrCloudTest {
+public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase {
@BeforeClass
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Wed Aug 8 18:15:57 2012
@@ -26,12 +26,11 @@ import org.apache.solr.client.solrj.embe
import org.apache.solr.common.SolrInputDocument;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Slow
-public class RecoveryZkTest extends FullSolrCloudTest {
+public class RecoveryZkTest extends AbstractFullDistribZkTestBase {
//private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain";
private static Logger log = LoggerFactory.getLogger(RecoveryZkTest.class);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Wed Aug 8 18:15:57 2012
@@ -38,15 +38,13 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
/**
* Test sync phase that occurs when Leader goes down and a new Leader is
* elected.
*/
@Slow
-@Ignore("ignore while i look into")
-public class SyncSliceTest extends FullSolrCloudTest {
+public class SyncSliceTest extends AbstractFullDistribZkTestBase {
@BeforeClass
public static void beforeSuperClass() {
@@ -88,7 +86,7 @@ public class SyncSliceTest extends FullS
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
del("*:*");
List<String> skipServers = new ArrayList<String>();
@@ -131,7 +129,7 @@ public class SyncSliceTest extends FullS
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
baseServer.request(request);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
checkShardConsistency(false, true);
@@ -161,7 +159,7 @@ public class SyncSliceTest extends FullS
// to talk to a downed node causes grief
waitToSeeDownInClusterState(leaderJetty, jetties);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
checkShardConsistency(false, true);
@@ -182,7 +180,7 @@ public class SyncSliceTest extends FullS
// give a moment to be sure it has started recovering
Thread.sleep(2000);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
waitForRecoveriesToFinish(false);
skipServers = getRandomOtherJetty(leaderJetty, null);
@@ -226,6 +224,7 @@ public class SyncSliceTest extends FullS
waitForRecoveriesToFinish(false);
checkShardConsistency(true, true);
+
}
private List<String> getRandomJetty() {
@@ -260,34 +259,6 @@ public class SyncSliceTest extends FullS
}
waitToSeeNotLive(cloudClient.getZkStateReader(), leaderJetty);
}
-
- private void waitForThingsToLevelOut() throws Exception {
- int cnt = 0;
- boolean retry = false;
- do {
- waitForRecoveriesToFinish(false);
-
- commit();
-
- updateMappingsFromZk(jettys, clients);
-
- Set<String> theShards = shardToJetty.keySet();
- String failMessage = null;
- for (String shard : theShards) {
- failMessage = checkShardConsistency(shard, false);
- }
-
- if (failMessage != null) {
- retry = true;
- } else {
- retry = false;
- }
-
- cnt++;
- if (cnt > 10) break;
- Thread.sleep(2000);
- } while (retry);
- }
protected void indexDoc(List<String> skipServers, Object... fields) throws IOException,
SolrServerException {
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Wed Aug 8 18:15:57 2012
@@ -65,10 +65,14 @@ public class CloudSolrServer extends Sol
// since the state shouldn't change often, should be very cheap reads
private volatile List<String> urlList;
+
private volatile List<String> leaderUrlList;
+ private volatile List<String> replicasList;
+
private volatile int lastClusterStateHashCode;
private final boolean updatesToLeaders;
+
/**
* @param zkHost The client endpoint of the zookeeper quorum containing the cloud state,
@@ -164,9 +168,11 @@ public class CloudSolrServer extends Sol
ClusterState clusterState = zkStateReader.getClusterState();
boolean sendToLeaders = false;
+ List<String> replicas = null;
if (request instanceof IsUpdateRequest && updatesToLeaders) {
sendToLeaders = true;
+ replicas = new ArrayList<String>();
}
SolrParams reqParams = request.getParams();
@@ -211,17 +217,22 @@ public class CloudSolrServer extends Sol
if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) {
String url = coreNodeProps.getCoreUrl();
urlList.add(url);
+ } else if (sendToLeaders) {
+ String url = coreNodeProps.getCoreUrl();
+ replicas.add(url);
}
}
}
}
if (sendToLeaders) {
this.leaderUrlList = urlList;
+ this.replicasList = replicas;
} else {
this.urlList = urlList;
}
this.lastClusterStateHashCode = clusterState.hashCode();
}
+
List<String> theUrlList;
if (sendToLeaders) {
theUrlList = new ArrayList<String>(leaderUrlList.size());
@@ -231,7 +242,14 @@ public class CloudSolrServer extends Sol
theUrlList.addAll(urlList);
}
Collections.shuffle(theUrlList, rand);
- //System.out.println("########################## MAKING REQUEST TO " + urlList);
+ if (replicas != null) {
+ ArrayList<String> theReplicas = new ArrayList<String>(replicasList.size());
+ theReplicas.addAll(replicasList);
+ Collections.shuffle(theReplicas, rand);
+
+ theUrlList.addAll(theReplicas);
+ }
+ //System.out.println("########################## MAKING REQUEST TO " + theUrlList);
LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, theUrlList);
LBHttpSolrServer.Rsp rsp = lbServer.request(req);
@@ -255,4 +273,16 @@ public class CloudSolrServer extends Sol
public LBHttpSolrServer getLbServer() {
return lbServer;
}
+
+ List<String> getUrlList() {
+ return urlList;
+ }
+
+ List<String> getLeaderUrlList() {
+ return leaderUrlList;
+ }
+
+ List<String> getReplicasList() {
+ return replicasList;
+ }
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java Wed Aug 8 18:15:57 2012
@@ -237,7 +237,7 @@ public class TestLBHttpSolrServer extend
}
public String getSchemaFile() {
- return "solrj/solr/conf/schema-replication1.xml";
+ return "solrj/solr/collection1/conf/schema-replication1.xml";
}
public String getConfDir() {
@@ -249,7 +249,7 @@ public class TestLBHttpSolrServer extend
}
public String getSolrConfigFile() {
- return "solrj/solr/conf/solrconfig-slave1.xml";
+ return "solrj/solr/collection1/conf/solrconfig-slave1.xml";
}
public void setUp() throws Exception {
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java Wed Aug 8 18:15:57 2012
@@ -1388,11 +1388,10 @@ public abstract class SolrTestCaseJ4 ext
return file;
} catch (Exception e) {
/* more friendly than NPE */
- throw new RuntimeException("Cannot find resource: " + name);
+ throw new RuntimeException("Cannot find resource: " + new File(name).getAbsolutePath());
}
}
- // TODO: use solr rather than solr/collection1
public static String TEST_HOME() {
return getFile("solr/collection1").getParent();
}
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java?rev=1370869&r1=1370868&r2=1370869&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java Wed Aug 8 18:15:57 2012
@@ -25,7 +25,7 @@ import java.io.File;
* @lucene.internal
*/
public class ExternalPaths {
- private static final String SOURCE_HOME = determineSourceHome();
+ public static final String SOURCE_HOME = determineSourceHome();
public static String WEBAPP_HOME = new File(SOURCE_HOME, "webapp/web").getAbsolutePath();
public static String EXAMPLE_HOME = new File(SOURCE_HOME, "example/solr").getAbsolutePath();
public static String EXAMPLE_MULTICORE_HOME = new File(SOURCE_HOME, "example/multicore").getAbsolutePath();