You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2015/10/27 17:37:32 UTC
svn commit: r1710844 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/test/org/apache/solr/cloud/ solr/test-framework/
solr/test-framework/src/java/org/apache/solr/cloud/
Author: cpoerschke
Date: Tue Oct 27 16:37:32 2015
New Revision: 1710844
URL: http://svn.apache.org/viewvc?rev=1710844&view=rev
Log:
SOLR-8196: TestMiniSolrCloudCluster.testStopAllStartAll case plus necessary MiniSolrCloudCluster tweak (merge in revision 1710824 from trunk)
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
lucene/dev/branches/branch_5x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1710844&r1=1710843&r2=1710844&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Tue Oct 27 16:37:32 2015
@@ -277,6 +277,9 @@ Other Changes
* SOLR-4854: Add a test to assert that [elevated] DocTransfer works correctly with javabin
response format. (Ray, shalin)
+* SOLR-8196: TestMiniSolrCloudCluster.testStopAllStartAll case plus necessary
+ MiniSolrCloudCluster tweak (Christine Poerschke)
+
================== 5.3.1 ==================
Bug Fixes
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java?rev=1710844&r1=1710843&r2=1710844&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java Tue Oct 27 16:37:32 2015
@@ -19,8 +19,12 @@ package org.apache.solr.cloud;
import java.io.File;
import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@@ -90,15 +94,26 @@ public class TestMiniSolrCloudCluster ex
}
private MiniSolrCloudCluster createMiniSolrCloudCluster() throws Exception {
+ return createMiniSolrCloudCluster(false);
+ }
+
+ private MiniSolrCloudCluster createMiniSolrCloudCluster(boolean multipleBaseDirs) throws Exception {
File solrXml = new File(SolrTestCaseJ4.TEST_HOME(), "solr-no-core.xml");
Builder jettyConfig = JettyConfig.builder();
jettyConfig.waitForLoadingCoresToFinish(null);
- MiniSolrCloudCluster miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir().toFile(), solrXml, jettyConfig.build());
- return miniCluster;
+ if (multipleBaseDirs) {
+ final File baseDirs[] = new File[NUM_SERVERS];
+ for (int ii = 0; ii < NUM_SERVERS; ++ii) {
+ baseDirs[ii] = createTempDir().toFile();
+ }
+ return new MiniSolrCloudCluster(baseDirs, solrXml, jettyConfig.build(), null);
+ } else {
+ return new MiniSolrCloudCluster(NUM_SERVERS, createTempDir().toFile(), solrXml, jettyConfig.build());
+ }
}
- private void createCollection(MiniSolrCloudCluster miniCluster, String collectionName, String createNodeSet, String asyncId) throws Exception {
+ private void createCollection(MiniSolrCloudCluster miniCluster, String collectionName, String createNodeSet, String asyncId, boolean persistIndex) throws Exception {
String configName = "solrCloudCollectionConfig";
File configDir = new File(SolrTestCaseJ4.TEST_HOME() + File.separator + "collection1" + File.separator + "conf");
miniCluster.uploadConfigDir(configDir, configName);
@@ -110,7 +125,7 @@ public class TestMiniSolrCloudCluster ex
// use non-test classes so RandomizedRunner isn't necessary
collectionProperties.put("solr.tests.mergePolicy", "org.apache.lucene.index.TieredMergePolicy");
collectionProperties.put("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
- collectionProperties.put("solr.directoryFactory", "solr.RAMDirectoryFactory");
+ collectionProperties.put("solr.directoryFactory", (persistIndex ? "solr.StandardDirectoryFactory" : "solr.RAMDirectoryFactory"));
miniCluster.createCollection(collectionName, NUM_SHARDS, REPLICATION_FACTOR, configName, createNodeSet, asyncId, collectionProperties);
}
@@ -141,7 +156,7 @@ public class TestMiniSolrCloudCluster ex
// create collection
final String asyncId = (random().nextBoolean() ? null : "asyncId("+collectionName+".create)="+random().nextInt());
- createCollection(miniCluster, collectionName, null, asyncId);
+ createCollection(miniCluster, collectionName, null, asyncId, random().nextBoolean());
if (asyncId != null) {
assertEquals("did not see async createCollection completion", "completed", AbstractFullDistribZkTestBase.getRequestStateAfterCompletion(asyncId, 330, cloudSolrClient));
}
@@ -295,7 +310,7 @@ public class TestMiniSolrCloudCluster ex
// create collection
final String asyncId = (random().nextBoolean() ? null : "asyncId("+collectionName+".create)="+random().nextInt());
- createCollection(miniCluster, collectionName, OverseerCollectionMessageHandler.CREATE_NODE_SET_EMPTY, asyncId);
+ createCollection(miniCluster, collectionName, OverseerCollectionMessageHandler.CREATE_NODE_SET_EMPTY, asyncId, random().nextBoolean());
if (asyncId != null) {
assertEquals("did not see async createCollection completion", "completed", AbstractFullDistribZkTestBase.getRequestStateAfterCompletion(asyncId, 330, cloudSolrClient));
}
@@ -322,6 +337,124 @@ public class TestMiniSolrCloudCluster ex
}
}
finally {
+ miniCluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testStopAllStartAll() throws Exception {
+
+ final String collectionName = "testStopAllStartAllCollection";
+
+ final MiniSolrCloudCluster miniCluster = createMiniSolrCloudCluster(true);
+
+ try {
+ assertNotNull(miniCluster.getZkServer());
+ List<JettySolrRunner> jettys = miniCluster.getJettySolrRunners();
+ assertEquals(NUM_SERVERS, jettys.size());
+ for (JettySolrRunner jetty : jettys) {
+ assertTrue(jetty.isRunning());
+ }
+
+ createCollection(miniCluster, collectionName, null, null, true);
+ final CloudSolrClient cloudSolrClient = miniCluster.getSolrClient();
+ cloudSolrClient.setDefaultCollection(collectionName);
+ final SolrQuery query = new SolrQuery("*:*");
+ final SolrInputDocument doc = new SolrInputDocument();
+
+ try (SolrZkClient zkClient = new SolrZkClient
+ (miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null);
+ ZkStateReader zkStateReader = new ZkStateReader(zkClient)) {
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330);
+
+ // modify collection
+ final int numDocs = 1 + random().nextInt(10);
+ for (int ii = 1; ii <= numDocs; ++ii) {
+ doc.setField("id", ""+ii);
+ cloudSolrClient.add(doc);
+ if (ii*2 == numDocs) cloudSolrClient.commit();
+ }
+ cloudSolrClient.commit();
+ // query collection
+ {
+ final QueryResponse rsp = cloudSolrClient.query(query);
+ assertEquals(numDocs, rsp.getResults().getNumFound());
+ }
+
+ // the test itself
+ zkStateReader.updateClusterState();
+ final ClusterState clusterState = zkStateReader.getClusterState();
+
+ final HashSet<Integer> leaderIndices = new HashSet<Integer>();
+ final HashSet<Integer> followerIndices = new HashSet<Integer>();
+ {
+ final HashMap<String,Boolean> shardLeaderMap = new HashMap<String,Boolean>();
+ for (final Slice slice : clusterState.getSlices(collectionName)) {
+ for (final Replica replica : slice.getReplicas()) {
+ shardLeaderMap.put(replica.getNodeName().replace("_solr", "/solr"), Boolean.FALSE);
+ }
+ shardLeaderMap.put(slice.getLeader().getNodeName().replace("_solr", "/solr"), Boolean.TRUE);
+ }
+ for (int ii = 0; ii < jettys.size(); ++ii) {
+ final URL jettyBaseUrl = jettys.get(ii).getBaseUrl();
+ final String jettyBaseUrlString = jettyBaseUrl.toString().substring((jettyBaseUrl.getProtocol() + "://").length());
+ final Boolean isLeader = shardLeaderMap.get(jettyBaseUrlString);
+ if (Boolean.TRUE.equals(isLeader)) {
+ leaderIndices.add(new Integer(ii));
+ } else if (Boolean.FALSE.equals(isLeader)) {
+ followerIndices.add(new Integer(ii));
+ } // else neither leader nor follower i.e. node without a replica (for our collection)
+ }
+ }
+ final List<Integer> leaderIndicesList = new ArrayList<Integer>(leaderIndices);
+ final List<Integer> followerIndicesList = new ArrayList<Integer>(followerIndices);
+
+ // first stop the followers (in no particular order)
+ Collections.shuffle(followerIndicesList, random());
+ for (Integer ii : followerIndicesList) {
+ if (!leaderIndices.contains(ii)) {
+ miniCluster.stopJettySolrRunner(jettys.get(ii.intValue()));
+ }
+ }
+
+ // then stop the leaders (again in no particular order)
+ Collections.shuffle(leaderIndicesList, random());
+ for (Integer ii : leaderIndicesList) {
+ miniCluster.stopJettySolrRunner(jettys.get(ii.intValue()));
+ }
+
+ // calculate restart order
+ final List<Integer> restartIndicesList = new ArrayList<Integer>();
+ Collections.shuffle(leaderIndicesList, random());
+ restartIndicesList.addAll(leaderIndicesList);
+ Collections.shuffle(followerIndicesList, random());
+ restartIndicesList.addAll(followerIndicesList);
+ if (random().nextBoolean()) Collections.shuffle(restartIndicesList, random());
+
+ // and then restart jettys in that order
+ for (Integer ii : restartIndicesList) {
+ final JettySolrRunner jetty = jettys.get(ii.intValue());
+ if (!jetty.isRunning()) {
+ miniCluster.startJettySolrRunner(jetty);
+ assertTrue(jetty.isRunning());
+ }
+ }
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330);
+
+ zkStateReader.updateClusterState();
+
+ // re-query collection
+ {
+ final QueryResponse rsp = cloudSolrClient.query(query);
+ assertEquals(numDocs, rsp.getResults().getNumFound());
+ }
+
+ // delete the collection we created earlier
+ miniCluster.deleteCollection(collectionName);
+ AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, zkStateReader, true, true, 330);
+ }
+ }
+ finally {
miniCluster.shutdown();
}
}
Modified: lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java?rev=1710844&r1=1710843&r2=1710844&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java (original)
+++ lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java Tue Oct 27 16:37:32 2015
@@ -67,6 +67,7 @@ public class MiniSolrCloudCluster {
private final boolean externalZkServer;
private final List<JettySolrRunner> jettys = new LinkedList<>();
private final File testDir;
+ private final File testDirs[];
private final CloudSolrClient solrClient;
private final JettyConfig jettyConfig;
@@ -141,13 +142,22 @@ public class MiniSolrCloudCluster {
* @throws Exception if there was an error starting the cluster
*/
public MiniSolrCloudCluster(int numServers, File baseDir, File solrXml, final JettyConfig jettyConfig, ZkTestServer zkTestServer) throws Exception {
+ this(numServers, baseDir, null, solrXml, jettyConfig, zkTestServer);
+ }
+
+ public MiniSolrCloudCluster(File[] baseDirs, File solrXml, final JettyConfig jettyConfig, ZkTestServer zkTestServer) throws Exception {
+ this(baseDirs.length, null, baseDirs, solrXml, jettyConfig, zkTestServer);
+ }
+
+ private MiniSolrCloudCluster(int numServers, File baseDir, File[] baseDirs, File solrXml, final JettyConfig jettyConfig, ZkTestServer zkTestServer) throws Exception {
this.testDir = baseDir;
+ this.testDirs = baseDirs;
this.jettyConfig = jettyConfig;
this.externalZkServer = zkTestServer != null;
if (!externalZkServer) {
- String zkDir = testDir.getAbsolutePath() + File.separator
+ String zkDir = (testDir != null ? testDir : testDirs[0]).getAbsolutePath() + File.separator
+ "zookeeper/server1/data";
zkTestServer = new ZkTestServer(zkDir);
zkTestServer.run();
@@ -167,10 +177,14 @@ public class MiniSolrCloudCluster {
List<Callable<JettySolrRunner>> startups = new ArrayList<>(numServers);
for (int i = 0; i < numServers; ++i) {
+ final Integer testDirsIdx = new Integer(i);
startups.add(new Callable<JettySolrRunner>() {
@Override
public JettySolrRunner call() throws Exception {
- return startJettySolrRunner(jettyConfig);
+ if (testDir != null)
+ return startJettySolrRunner(jettyConfig);
+ else
+ return startJettySolrRunner(testDirsIdx, jettyConfig.context, jettyConfig);
}
});
}
@@ -283,9 +297,12 @@ public class MiniSolrCloudCluster {
* @return a JettySolrRunner
*/
public JettySolrRunner startJettySolrRunner(String hostContext, JettyConfig config) throws Exception {
+ return startJettySolrRunner(null, hostContext, config);
+ }
+ public JettySolrRunner startJettySolrRunner(Integer testDirsIdx, String hostContext, JettyConfig config) throws Exception {
String context = getHostContextSuitableForServletContext(hostContext);
JettyConfig newConfig = JettyConfig.builder(config).setContext(context).build();
- JettySolrRunner jetty = new JettySolrRunner(testDir.getAbsolutePath(), newConfig);
+ JettySolrRunner jetty = new JettySolrRunner((testDirsIdx != null ? testDirs[testDirsIdx.intValue()] : testDir).getAbsolutePath(), newConfig);
jetty.start();
jettys.add(jetty);
return jetty;
@@ -312,6 +329,11 @@ public class MiniSolrCloudCluster {
return jetty;
}
+ protected JettySolrRunner startJettySolrRunner(JettySolrRunner jetty) throws Exception {
+ jetty.start();
+ return jetty;
+ }
+
protected JettySolrRunner stopJettySolrRunner(JettySolrRunner jetty) throws Exception {
jetty.stop();
return jetty;