You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/03/19 04:11:03 UTC
svn commit: r1579116 - in /lucene/dev/trunk/solr: CHANGES.txt
core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
Author: markrmiller
Date: Wed Mar 19 03:11:03 2014
New Revision: 1579116
URL: http://svn.apache.org/r1579116
Log:
SOLR-5865: Provide a MiniSolrCloudCluster to enable easier testing.
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java (with props)
lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1579116&r1=1579115&r2=1579116&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Mar 19 03:11:03 2014
@@ -134,6 +134,9 @@ New Features
* SOLR-1604: Wildcards, ORs etc inside Phrase Queries. (Ahmet Arslan via Erick Erickson)
+* SOLR-5865: Provide a MiniSolrCloudCluster to enable easier testing.
+ (Greg Chanan via Mark Miller)
+
Bug Fixes
----------------------
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java?rev=1579116&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java Wed Mar 19 03:11:03 2014
@@ -0,0 +1,213 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.SolrTestCaseJ4;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Test of the MiniSolrCloudCluster functionality. This doesn't derive from
+ * LuceneTestCase, as the MiniSolrCloudCluster is designed to be used outside of the
+ * lucene test hierarchy.
+ */
+@Ignore
+public class TestMiniSolrCloudCluster {
+
+ private static Logger log = LoggerFactory.getLogger(MiniSolrCloudCluster.class);
+ private static final int NUM_SERVERS = 5;
+ private static final int NUM_SHARDS = 2;
+ private static final int REPLICATION_FACTOR = 2;
+ private static final Random RANDOM = new Random();
+ private static MiniSolrCloudCluster miniCluster;
+
+ @BeforeClass
+ public static void startup() throws Exception {
+ String testHome = SolrTestCaseJ4.TEST_HOME();
+ miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, new File(testHome, "solr-no-core.xml"),
+ null, null);
+ }
+
+ @AfterClass
+ public static void shutdown() throws Exception {
+ if (miniCluster != null) {
+ miniCluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testBasics() throws Exception {
+ assertNotNull(miniCluster.getZkServer());
+ List<JettySolrRunner> jettys = miniCluster.getJettySolrRunners();
+ assertEquals(NUM_SERVERS, jettys.size());
+ for (JettySolrRunner jetty : jettys) {
+ assertTrue(jetty.isRunning());
+ }
+
+ // shut down a server
+ JettySolrRunner stoppedServer = miniCluster.stopJettySolrRunner(0);
+ assertTrue(stoppedServer.isStopped());
+ assertEquals(NUM_SERVERS - 1, miniCluster.getJettySolrRunners().size());
+
+ // create a server
+ JettySolrRunner startedServer = miniCluster.startJettySolrRunner(null, null, null);
+ assertTrue(startedServer.isRunning());
+ assertEquals(NUM_SERVERS, miniCluster.getJettySolrRunners().size());
+
+ CloudSolrServer cloudSolrServer = null;
+ SolrZkClient zkClient = null;
+ try {
+ cloudSolrServer = new CloudSolrServer(miniCluster.getZkServer().getZkAddress(), RANDOM.nextBoolean());
+ cloudSolrServer.connect();
+ zkClient = new SolrZkClient(miniCluster.getZkServer().getZkAddress(),
+ AbstractZkTestCase.TIMEOUT, 45000, null);
+
+ // create collection
+ String collectionName = "testSolrCloudCollection";
+ String configName = "solrCloudCollectionConfig";
+ System.setProperty("solr.tests.mergePolicy", "org.apache.lucene.index.TieredMergePolicy");
+ uploadConfigToZk(SolrTestCaseJ4.TEST_HOME() + File.separator + "collection1" + File.separator + "conf", configName);
+ createCollection(cloudSolrServer, collectionName, NUM_SHARDS, REPLICATION_FACTOR, configName);
+
+ // modify/query collection
+ cloudSolrServer.setDefaultCollection(collectionName);
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", "1");
+
+ cloudSolrServer.add(doc);
+ cloudSolrServer.commit();
+ SolrQuery query = new SolrQuery();
+ query.setQuery("*:*");
+ QueryResponse rsp = cloudSolrServer.query(query);
+ assertEquals(1, rsp.getResults().getNumFound());
+
+ // remove a server not hosting any replicas
+ ZkStateReader zkStateReader = new ZkStateReader(zkClient);
+ zkStateReader.updateClusterState(true);
+ ClusterState clusterState = zkStateReader.getClusterState();
+ HashMap<String, JettySolrRunner> jettyMap = new HashMap<String, JettySolrRunner>();
+ for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) {
+ String key = jetty.getBaseUrl().toString().substring((jetty.getBaseUrl().getProtocol() + "://").length());
+ jettyMap.put(key, jetty);
+ }
+ Collection<Slice> slices = clusterState.getSlices(collectionName);
+ // track the servers not host repliacs
+ for (Slice slice : slices) {
+ jettyMap.remove(slice.getLeader().getNodeName().replace("_solr", "/solr"));
+ for (Replica replica : slice.getReplicas()) {
+ jettyMap.remove(replica.getNodeName().replace("_solr", "/solr"));
+ }
+ }
+ assertTrue("Expected to find a node without a replica", jettyMap.size() > 0);
+ JettySolrRunner jettyToStop = jettyMap.entrySet().iterator().next().getValue();
+ jettys = miniCluster.getJettySolrRunners();
+ for (int i = 0; i < jettys.size(); ++i) {
+ if (jettys.get(i).equals(jettyToStop)) {
+ miniCluster.stopJettySolrRunner(i);
+ assertEquals(NUM_SERVERS - 1, miniCluster.getJettySolrRunners().size());
+ }
+ }
+ } finally {
+ if (cloudSolrServer != null) {
+ cloudSolrServer.shutdown();
+ }
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ }
+ }
+
+ protected void uploadConfigToZk(String configDir, String configName) throws Exception {
+ // override settings in the solrconfig include
+ System.setProperty("solr.tests.maxBufferedDocs", "100000");
+ System.setProperty("solr.tests.maxIndexingThreads", "-1");
+ System.setProperty("solr.tests.ramBufferSizeMB", "100");
+ // use non-test classes so RandomizedRunner isn't necessary
+ System.setProperty("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
+ System.setProperty("solr.directoryFactory", "solr.RAMDirectoryFactory");
+
+ SolrZkClient zkClient = null;
+ try {
+ zkClient = new SolrZkClient(miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null);
+ uploadConfigFileToZk(zkClient, configName, "solrconfig.xml", new File(configDir, "solrconfig-tlog.xml"));
+ uploadConfigFileToZk(zkClient, configName, "schema.xml", new File(configDir, "schema.xml"));
+ uploadConfigFileToZk(zkClient, configName, "solrconfig.snippet.randomindexconfig.xml",
+ new File(configDir, "solrconfig.snippet.randomindexconfig.xml"));
+ uploadConfigFileToZk(zkClient, configName, "currency.xml", new File(configDir, "currency.xml"));
+ uploadConfigFileToZk(zkClient, configName, "mapping-ISOLatin1Accent.txt",
+ new File(configDir, "mapping-ISOLatin1Accent.txt"));
+ uploadConfigFileToZk(zkClient, configName, "old_synonyms.txt", new File(configDir, "old_synonyms.txt"));
+ uploadConfigFileToZk(zkClient, configName, "open-exchange-rates.json",
+ new File(configDir, "open-exchange-rates.json"));
+ uploadConfigFileToZk(zkClient, configName, "protwords.txt", new File(configDir, "protwords.txt"));
+ uploadConfigFileToZk(zkClient, configName, "stopwords.txt", new File(configDir, "stopwords.txt"));
+ uploadConfigFileToZk(zkClient, configName, "synonyms.txt", new File(configDir, "synonyms.txt"));
+ } finally {
+ if (zkClient != null) zkClient.close();
+ }
+ }
+
+ protected void uploadConfigFileToZk(SolrZkClient zkClient, String configName, String nameInZk, File file)
+ throws Exception {
+ zkClient.makePath(ZkController.CONFIGS_ZKNODE + "/" + configName + "/" + nameInZk, file, false, true);
+ }
+
+ protected NamedList<Object> createCollection(CloudSolrServer server, String name, int numShards,
+ int replicationFactor, String configName) throws Exception {
+ ModifiableSolrParams modParams = new ModifiableSolrParams();
+ modParams.set(CoreAdminParams.ACTION, CollectionAction.CREATE.name());
+ modParams.set("name", name);
+ modParams.set("numShards", numShards);
+ modParams.set("replicationFactor", replicationFactor);
+ modParams.set("collection.configName", configName);
+ QueryRequest request = new QueryRequest(modParams);
+ request.setPath("/admin/collections");
+ return server.request(request);
+ }
+}
Added: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java?rev=1579116&view=auto
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java (added)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java Wed Mar 19 03:11:03 2014
@@ -0,0 +1,150 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs;
+
+import org.eclipse.jetty.servlet.ServletHolder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.SortedMap;
+
+import com.google.common.io.Files;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MiniSolrCloudCluster {
+ private static Logger log = LoggerFactory.getLogger(MiniSolrCloudCluster.class);
+
+ private ZkTestServer zkServer;
+ private List<JettySolrRunner> jettys;
+ private File testDir;
+
+ /**
+ * "Mini" SolrCloud cluster to be used for testing
+ * @param numServers number of Solr servers to start
+ * @param hostContext context path of Solr servers used by Jetty
+ * @param solrXml solr.xml file to be uploaded to ZooKeeper
+ * @param extraServlets Extra servlets to be started by Jetty
+ * @param extraRequestFilters extra filters to be started by Jetty
+ */
+ public MiniSolrCloudCluster(int numServers, String hostContext, File solrXml,
+ SortedMap<ServletHolder, String> extraServlets,
+ SortedMap<Class, String> extraRequestFilters) throws Exception {
+ testDir = Files.createTempDir();
+
+ String zkDir = testDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+ zkServer = new ZkTestServer(zkDir);
+ zkServer.run();
+
+ SolrZkClient zkClient = null;
+ InputStream is = null;
+ try {
+ zkClient = new SolrZkClient(zkServer.getZkHost(),
+ AbstractZkTestCase.TIMEOUT, 45000, null);
+ zkClient.makePath("/solr", false, true);
+ is = new FileInputStream(solrXml);
+ zkClient.create("/solr/solr.xml", IOUtils.toByteArray(is),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true);
+ } finally {
+ IOUtils.closeQuietly(is);
+ if (zkClient != null) zkClient.close();
+ }
+
+ // tell solr to look in zookeeper for solr.xml
+ System.setProperty("solr.solrxml.location","zookeeper");
+ System.setProperty("zkHost", zkServer.getZkAddress());
+
+ jettys = new LinkedList<JettySolrRunner>();
+ for (int i = 0; i < numServers; ++i) {
+ startJettySolrRunner(hostContext, extraServlets, extraRequestFilters);
+ }
+ }
+
+ /**
+ * @return ZooKeeper server used by the MiniCluster
+ */
+ public ZkTestServer getZkServer() {
+ return zkServer;
+ }
+
+ /**
+ * @return Unmodifiable list of all the currently started Solr Jettys.
+ */
+ public List<JettySolrRunner> getJettySolrRunners() {
+ return Collections.unmodifiableList(jettys);
+ }
+
+ /**
+ * Start a new Solr instance
+ * @param hostContext context path of Solr servers used by Jetty
+ * @param extraServlets Extra servlets to be started by Jetty
+ * @param extraRequestFilters extra filters to be started by Jetty
+ * @return new Solr instance
+ */
+ public JettySolrRunner startJettySolrRunner(String hostContext,
+ SortedMap<ServletHolder, String> extraServlets,
+ SortedMap<Class, String> extraRequestFilters) throws Exception {
+ String context = getHostContextSuitableForServletContext(hostContext);
+ JettySolrRunner jetty = new JettySolrRunner(testDir.getAbsolutePath(), context, 0, null, null,
+ true, extraServlets, null, extraRequestFilters);
+ jetty.start();
+ jettys.add(jetty);
+ return jetty;
+ }
+
+ /**
+ * Stop a Solr instance
+ * @param index the index of node in collection returned by {@link #getJettySolrRunners()}
+ * @return the shut down node
+ */
+ public JettySolrRunner stopJettySolrRunner(int index) throws Exception {
+ JettySolrRunner jetty = jettys.get(index);
+ jetty.stop();
+ jettys.remove(index);
+ return jetty;
+ }
+
+ /**
+ * Shut down the cluster, including all Solr nodes and ZooKeeper
+ */
+ public void shutdown() throws Exception {
+ for (int i = jettys.size() - 1; i >= 0; --i) {
+ stopJettySolrRunner(i);
+ }
+ zkServer.shutdown();
+ }
+
+ private static String getHostContextSuitableForServletContext(String ctx) {
+ if (ctx == null || "".equals(ctx)) ctx = "/solr";
+ if (ctx.endsWith("/")) ctx = ctx.substring(0,ctx.length()-1);;
+ if (!ctx.startsWith("/")) ctx = "/" + ctx;
+ return ctx;
+ }
+}
\ No newline at end of file