You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/07/24 09:21:32 UTC
svn commit: r1364924 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/test/org/apache/solr/cloud/
Author: markrmiller
Date: Tue Jul 24 07:21:31 2012
New Revision: 1364924
URL: http://svn.apache.org/viewvc?rev=1364924&view=rev
Log:
simplify + improve test infra
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java?rev=1364924&r1=1364923&r2=1364924&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java Tue Jul 24 07:21:31 2012
@@ -71,10 +71,8 @@ public class ChaosMonkey {
public ChaosMonkey(ZkTestServer zkServer, ZkStateReader zkStateReader,
String collection, Map<String,List<CloudJettyRunner>> shardToJetty,
- Map<String,List<SolrServer>> shardToClient,
Map<String,CloudJettyRunner> shardToLeaderJetty) {
this.shardToJetty = shardToJetty;
- this.shardToClient = shardToClient;
this.shardToLeaderJetty = shardToLeaderJetty;
this.zkServer = zkServer;
this.zkStateReader = zkStateReader;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1364924&r1=1364923&r2=1364924&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Tue Jul 24 07:21:31 2012
@@ -98,15 +98,12 @@ public class FullSolrCloudTest extends A
protected volatile CloudSolrServer cloudClient;
- protected Map<JettySolrRunner,ZkNodeProps> jettyToInfo = new HashMap<JettySolrRunner,ZkNodeProps>();
- protected Map<CloudSolrServerClient,ZkNodeProps> clientToInfo = new HashMap<CloudSolrServerClient,ZkNodeProps>();
- protected Map<String,List<SolrServer>> shardToClient = new HashMap<String,List<SolrServer>>();
+ protected List<CloudJettyRunner> cloudJettys = new ArrayList<CloudJettyRunner>();
protected Map<String,List<CloudJettyRunner>> shardToJetty = new HashMap<String,List<CloudJettyRunner>>();
private AtomicInteger jettyIntCntr = new AtomicInteger(0);
protected ChaosMonkey chaosMonkey;
protected volatile ZkStateReader zkStateReader;
- protected Map<String,SolrServer> shardToLeaderClient = new HashMap<String,SolrServer>();
protected Map<String,CloudJettyRunner> shardToLeaderJetty = new HashMap<String,CloudJettyRunner>();
static class CloudJettyRunner {
@@ -115,12 +112,14 @@ public class FullSolrCloudTest extends A
String coreNodeName;
String url;
CloudSolrServerClient client;
+ public ZkNodeProps info;
}
static class CloudSolrServerClient {
SolrServer solrClient;
String shardName;
int port;
+ public ZkNodeProps info;
public CloudSolrServerClient() {}
@@ -191,7 +190,7 @@ public class FullSolrCloudTest extends A
}
chaosMonkey = new ChaosMonkey(zkServer, zkStateReader,
- DEFAULT_COLLECTION, shardToJetty, shardToClient,
+ DEFAULT_COLLECTION, shardToJetty,
shardToLeaderJetty);
}
@@ -337,9 +336,8 @@ public class FullSolrCloudTest extends A
protected void updateMappingsFromZk(List<JettySolrRunner> jettys,
List<SolrServer> clients) throws Exception {
zkStateReader.updateCloudState(true);
- shardToClient.clear();
+ cloudJettys.clear();
shardToJetty.clear();
- jettyToInfo.clear();
CloudState cloudState = zkStateReader.getCloudState();
Map<String,Slice> slices = cloudState.getSlices(DEFAULT_COLLECTION);
@@ -349,6 +347,7 @@ public class FullSolrCloudTest extends A
+ DEFAULT_COLLECTION + " in " + cloudState.getCollections());
}
+ List<CloudSolrServerClient> theClients = new ArrayList<CloudSolrServerClient>();
for (SolrServer client : clients) {
// find info for this client in zk
nextClient:
@@ -364,34 +363,20 @@ public class FullSolrCloudTest extends A
csc.solrClient = client;
csc.port = port;
csc.shardName = shard.getValue().get(ZkStateReader.NODE_NAME_PROP);
- boolean isLeader = shard.getValue().containsKey(
- ZkStateReader.LEADER_PROP);
- clientToInfo.put(csc, shard.getValue());
- List<SolrServer> list = shardToClient.get(slice.getKey());
- if (list == null) {
- list = new ArrayList<SolrServer>();
- shardToClient.put(slice.getKey(), list);
- }
- list.add(client);
+ csc.info = shard.getValue();
+
+ theClients .add(csc);
- if (isLeader) {
- shardToLeaderClient.put(slice.getKey(), client);
- }
break nextClient;
}
}
}
}
-
- for (Map.Entry<String,Slice> slice : slices.entrySet()) {
- // check that things look right
- assertEquals(slice.getValue().getShards().size(), shardToClient.get(slice.getKey()).size());
- }
for (JettySolrRunner jetty : jettys) {
int port = jetty.getLocalPort();
if (port == -1) {
- continue; // If we cannot get the port, this jetty is down
+ throw new RuntimeException("Cannot find the port for jetty");
}
nextJetty:
@@ -399,7 +384,6 @@ public class FullSolrCloudTest extends A
Map<String,ZkNodeProps> theShards = slice.getValue().getShards();
for (Map.Entry<String,ZkNodeProps> shard : theShards.entrySet()) {
if (shard.getKey().contains(":" + port + "_")) {
- jettyToInfo.put(jetty, shard.getValue());
List<CloudJettyRunner> list = shardToJetty.get(slice.getKey());
if (list == null) {
list = new ArrayList<CloudJettyRunner>();
@@ -409,14 +393,16 @@ public class FullSolrCloudTest extends A
ZkStateReader.LEADER_PROP);
CloudJettyRunner cjr = new CloudJettyRunner();
cjr.jetty = jetty;
+ cjr.info = shard.getValue();
cjr.nodeName = shard.getValue().get(ZkStateReader.NODE_NAME_PROP);
cjr.coreNodeName = shard.getKey();
cjr.url = shard.getValue().get(ZkStateReader.BASE_URL_PROP) + "/" + shard.getValue().get(ZkStateReader.CORE_NAME_PROP);
- cjr.client = findClientByPort(port);
+ cjr.client = findClientByPort(port, theClients);
list.add(cjr);
if (isLeader) {
shardToLeaderJetty.put(slice.getKey(), cjr);
}
+ cloudJettys.add(cjr);
break nextJetty;
}
}
@@ -435,8 +421,7 @@ public class FullSolrCloudTest extends A
}
}
- private CloudSolrServerClient findClientByPort(int port) {
- Set<CloudSolrServerClient> theClients = clientToInfo.keySet();
+ private CloudSolrServerClient findClientByPort(int port, List<CloudSolrServerClient> theClients) {
for (CloudSolrServerClient client : theClients) {
if (client.port == port) {
return client;
@@ -661,9 +646,9 @@ public class FullSolrCloudTest extends A
// new server should be part of first shard
// how many docs are on the new shard?
- for (SolrServer client : shardToClient.get("shard1")) {
+ for (CloudJettyRunner cjetty : shardToJetty.get("shard1")) {
if (VERBOSE) System.err.println("total:"
- + client.query(new SolrQuery("*:*")).getResults().getNumFound());
+ + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound());
}
checkShardConsistency("shard1");
@@ -687,15 +672,21 @@ public class FullSolrCloudTest extends A
commit();
- long deadShardCount = shardToClient.get(SHARD2).get(0).query(query).getResults().getNumFound();
-
+ long deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
+ .query(query).getResults().getNumFound();
+
query("q", "*:*", "sort", "n_tl1 desc");
// kill a shard
CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD2, 0);
cloudClient.connect();
int tries = 0;
- while (cloudClient.getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(shardToClient.get(SHARD2).get(0))).get(ZkStateReader.NODE_NAME_PROP))) {
+ while (cloudClient
+ .getZkStateReader()
+ .getCloudState()
+ .liveNodesContain(
+ shardToJetty.get(SHARD2).get(0).info
+ .get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 60) {
fail("Shard still reported as live in zk");
}
@@ -724,7 +715,15 @@ public class FullSolrCloudTest extends A
// because on some systems (especially freebsd w/ blackhole enabled), trying
// to talk to a downed node causes grief
tries = 0;
- while (((SolrDispatchFilter) shardToJetty.get(SHARD2).get(1).jetty.getDispatchFilter().getFilter()).getCores().getZkController().getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(shardToClient.get(SHARD2).get(0))).get(ZkStateReader.NODE_NAME_PROP))) {
+ while (((SolrDispatchFilter) shardToJetty.get(SHARD2).get(1).jetty
+ .getDispatchFilter().getFilter())
+ .getCores()
+ .getZkController()
+ .getZkStateReader()
+ .getCloudState()
+ .liveNodesContain(
+ shardToJetty.get(SHARD2).get(0).info
+ .get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 120) {
fail("Shard still reported as live in zk");
}
@@ -733,7 +732,7 @@ public class FullSolrCloudTest extends A
long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
- index_specific(shardToClient.get(SHARD2).get(1), id, 1000, i1, 108, t1,
+ index_specific(shardToJetty.get(SHARD2).get(1).client.solrClient, id, 1000, i1, 108, t1,
"specific doc!");
commit();
@@ -798,7 +797,8 @@ public class FullSolrCloudTest extends A
waitForRecoveriesToFinish(false);
- deadShardCount = shardToClient.get(SHARD2).get(0).query(query).getResults().getNumFound();
+ deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
+ .query(query).getResults().getNumFound();
// if we properly recovered, we should now have the couple missing docs that
// came in while shard was down
checkShardConsistency(true, false);
@@ -1025,10 +1025,10 @@ public class FullSolrCloudTest extends A
protected String checkShardConsistency(String shard, boolean verbose)
throws Exception {
- List<SolrServer> solrClients = shardToClient.get(shard);
- if (solrClients == null) {
+ List<CloudJettyRunner> solrJetties = shardToJetty.get(shard);
+ if (solrJetties == null) {
throw new RuntimeException("shard not found:" + shard + " keys:"
- + shardToClient.keySet());
+ + shardToJetty.keySet());
}
long num = -1;
long lastNum = -1;
@@ -1040,18 +1040,18 @@ public class FullSolrCloudTest extends A
"The client count does not match up with the shard count for slice:"
+ shard,
zkStateReader.getCloudState().getSlice(DEFAULT_COLLECTION, shard)
- .getShards().size(), solrClients.size());
+ .getShards().size(), solrJetties.size());
SolrServer lastClient = null;
- for (SolrServer client : solrClients) {
- ZkNodeProps props = clientToInfo.get(new CloudSolrServerClient(client));
+ for (CloudJettyRunner cjetty : solrJetties) {
+ ZkNodeProps props = cjetty.info;
if (verbose) System.err.println("client" + cnt++);
if (verbose) System.err.println("PROPS:" + props);
try {
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
- num = client.query(query).getResults().getNumFound();
+ num = cjetty.client.solrClient.query(query).getResults().getNumFound();
} catch (SolrServerException e) {
if (verbose) System.err.println("error contacting client: "
+ e.getMessage() + "\n");
@@ -1076,7 +1076,7 @@ public class FullSolrCloudTest extends A
if (active && live) {
if (lastNum > -1 && lastNum != num && failMessage == null) {
failMessage = shard + " is not consistent. Got " + lastNum + " from " + lastClient + "lastClient"
- + " and got " + num + " from " + client;
+ + " and got " + num + " from " + cjetty.url;
if (verbose || true) {
System.err.println("######" + failMessage);
@@ -1087,14 +1087,14 @@ public class FullSolrCloudTest extends A
query.set("sort","id asc");
SolrDocumentList lst1 = lastClient.query(query).getResults();
- SolrDocumentList lst2 = client.query(query).getResults();
+ SolrDocumentList lst2 = cjetty.client.solrClient.query(query).getResults();
- showDiff(lst1, lst2, lastClient.toString(), client.toString());
+ showDiff(lst1, lst2, lastClient.toString(), cjetty.client.solrClient.toString());
}
}
lastNum = num;
- lastClient = client;
+ lastClient = cjetty.client.solrClient;
}
}
return failMessage;
@@ -1141,7 +1141,7 @@ public class FullSolrCloudTest extends A
updateMappingsFromZk(jettys, clients);
- Set<String> theShards = shardToClient.keySet();
+ Set<String> theShards = shardToJetty.keySet();
String failMessage = null;
for (String shard : theShards) {
String shardFailMessage = checkShardConsistency(shard, verbose);
@@ -1156,15 +1156,15 @@ public class FullSolrCloudTest extends A
if (checkVsControl) {
// now check that the right # are on each shard
- theShards = shardToClient.keySet();
+ theShards = shardToJetty.keySet();
int cnt = 0;
for (String s : theShards) {
- int times = shardToClient.get(s).size();
+ int times = shardToJetty.get(s).size();
for (int i = 0; i < times; i++) {
try {
- SolrServer client = shardToClient.get(s).get(i);
- ZkNodeProps props = clientToInfo.get(new CloudSolrServerClient(
- client));
+ CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
+ ZkNodeProps props = cjetty.info;
+ SolrServer client = cjetty.client.solrClient;
boolean active = props.get(ZkStateReader.STATE_PROP).equals(
ZkStateReader.ACTIVE);
if (active) {
@@ -1196,7 +1196,8 @@ public class FullSolrCloudTest extends A
}
private SolrServer getClient(String nodeName) {
- for (CloudSolrServerClient client : clientToInfo.keySet()) {
+ for (CloudJettyRunner cjetty : cloudJettys) {
+ CloudSolrServerClient client = cjetty.client;
if (client.shardName.equals(nodeName)) {
return client.solrClient;
}
@@ -1232,12 +1233,13 @@ public class FullSolrCloudTest extends A
+ DEFAULT_COLLECTION + " in " + cloudState.getCollections());
}
- for (SolrServer client : clients) {
+ for (CloudJettyRunner cjetty : cloudJettys) {
+ CloudSolrServerClient client = cjetty.client;
for (Map.Entry<String,Slice> slice : slices.entrySet()) {
Map<String,ZkNodeProps> theShards = slice.getValue().getShards();
for (Map.Entry<String,ZkNodeProps> shard : theShards.entrySet()) {
String shardName = new URI(
- ((HttpSolrServer) client).getBaseURL()).getPort()
+ ((HttpSolrServer) client.solrClient).getBaseURL()).getPort()
+ "_solr_";
if (verbose && shard.getKey().endsWith(shardName)) {
System.err.println("shard:" + slice.getKey());
@@ -1247,13 +1249,14 @@ public class FullSolrCloudTest extends A
}
long count = 0;
- String currentState = clientToInfo.get(new CloudSolrServerClient(client))
- .get(ZkStateReader.STATE_PROP);
- if (currentState != null && currentState.equals(ZkStateReader.ACTIVE) && zkStateReader.getCloudState().liveNodesContain(clientToInfo.get(new CloudSolrServerClient(client))
- .get(ZkStateReader.NODE_NAME_PROP))) {
+ String currentState = cjetty.info.get(ZkStateReader.STATE_PROP);
+ if (currentState != null
+ && currentState.equals(ZkStateReader.ACTIVE)
+ && zkStateReader.getCloudState().liveNodesContain(
+ cjetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
- count = client.query(query).getResults().getNumFound();
+ count = client.solrClient.query(query).getResults().getNumFound();
}
if (verbose) System.err.println("client docs:" + count + "\n\n");
@@ -1365,7 +1368,7 @@ public class FullSolrCloudTest extends A
updateMappingsFromZk(jettys, clients);
- Set<String> theShards = shardToClient.keySet();
+ Set<String> theShards = shardToJetty.keySet();
String failMessage = null;
for (String shard : theShards) {
failMessage = checkShardConsistency(shard, false);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1364924&r1=1364923&r2=1364924&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Tue Jul 24 07:21:31 2012
@@ -100,8 +100,8 @@ public class RecoveryZkTest extends Full
checkShardConsistency("shard1", false);
SolrQuery query = new SolrQuery("*:*");
query.setParam("distrib", "false");
- long client1Docs = shardToClient.get("shard1").get(0).query(query).getResults().getNumFound();
- long client2Docs = shardToClient.get("shard1").get(1).query(query).getResults().getNumFound();
+ long client1Docs = shardToJetty.get("shard1").get(0).client.solrClient.query(query).getResults().getNumFound();
+ long client2Docs = shardToJetty.get("shard1").get(1).client.solrClient.query(query).getResults().getNumFound();
assertTrue(client1Docs > 0);
assertEquals(client1Docs, client2Docs);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1364924&r1=1364923&r2=1364924&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Tue Jul 24 07:21:31 2012
@@ -26,13 +26,10 @@ import java.util.Set;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.cloud.FullSolrCloudTest.CloudSolrServerClient;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
@@ -122,7 +119,8 @@ public class SyncSliceTest extends FullS
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
- String baseUrl = ((HttpSolrServer) shardToClient.get("shard1").get(2)).getBaseURL();
+ String baseUrl = ((HttpSolrServer) shardToJetty.get("shard1").get(2).client.solrClient)
+ .getBaseURL();
baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
@@ -145,21 +143,28 @@ public class SyncSliceTest extends FullS
// kill the leader - new leader could have all the docs or be missing one
CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1");
- CloudSolrServerClient leaderClient = leaderJetty.client;
- Set<JettySolrRunner> jetties = new HashSet<JettySolrRunner>();
+
+ Set<CloudJettyRunner> jetties = new HashSet<CloudJettyRunner>();
for (int i = 0; i < shardCount; i++) {
- jetties.add(shardToJetty.get("shard1").get(i).jetty);
+ jetties.add(shardToJetty.get("shard1").get(i));
}
jetties.remove(leaderJetty);
chaosMonkey.killJetty(leaderJetty);
- JettySolrRunner upJetty = jetties.iterator().next();
+ CloudJettyRunner upJetty = jetties.iterator().next();
// we are careful to make sure the downed node is no longer in the state,
// because on some systems (especially freebsd w/ blackhole enabled), trying
// to talk to a downed node causes grief
+ assertNotNull(upJetty.jetty.getDispatchFilter());
+ assertNotNull(upJetty.jetty.getDispatchFilter());
+ assertNotNull(upJetty.jetty.getDispatchFilter().getFilter());
+
+
int tries = 0;
- while (((SolrDispatchFilter) upJetty.getDispatchFilter().getFilter()).getCores().getZkController().getZkStateReader().getCloudState().liveNodesContain(clientToInfo.get(leaderClient).get(ZkStateReader.NODE_NAME_PROP))) {
+ while (((SolrDispatchFilter) upJetty.jetty.getDispatchFilter().getFilter())
+ .getCores().getZkController().getZkStateReader().getCloudState()
+ .liveNodesContain(leaderJetty.info.get(ZkStateReader.NODE_NAME_PROP))) {
if (tries++ == 120) {
fail("Shard still reported as live in zk");
}
@@ -184,7 +189,7 @@ public class SyncSliceTest extends FullS
updateMappingsFromZk(jettys, clients);
- Set<String> theShards = shardToClient.keySet();
+ Set<String> theShards = shardToJetty.keySet();
String failMessage = null;
for (String shard : theShards) {
failMessage = checkShardConsistency(shard, false);