You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2017/04/26 23:28:39 UTC
[1/2] lucene-solr:jira/solr-10233: Sometimes use legacyCloud in tests
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-10233 a217dfaaf -> 304add6f6
Sometimes use legacyCloud in tests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0330b4ab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0330b4ab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0330b4ab
Branch: refs/heads/jira/solr-10233
Commit: 0330b4abe5785e509b29d3bc7f461c4e57d153f7
Parents: a217dfa
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed Apr 26 16:21:40 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed Apr 26 16:21:40 2017 -0700
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/cloud/ZkController.java | 1 +
.../src/test/org/apache/solr/cloud/TestAppendReplica.java | 4 +++-
.../src/test/org/apache/solr/cloud/TestPassiveReplica.java | 8 +++-----
3 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0330b4ab/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 55dd9a0..b239dd2 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1230,6 +1230,7 @@ public class ZkController {
props.put(ZkStateReader.NODE_NAME_PROP, getNodeName());
props.put(ZkStateReader.SHARD_ID_PROP, cd.getCloudDescriptor().getShardId());
props.put(ZkStateReader.COLLECTION_PROP, collection);
+ props.put(ZkStateReader.REPLICA_TYPE, cd.getCloudDescriptor().getReplicaType().toString());
if (numShards != null) {
props.put(ZkStateReader.NUM_SHARDS_PROP, numShards.toString());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0330b4ab/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
index fe353d2..a7f3042 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
@@ -82,7 +82,9 @@ public class TestAppendReplica extends SolrCloudTestCase {
configureCluster(2) // 2 + random().nextInt(3)
.addConfig("conf", configset("cloud-minimal-inplace-updates"))
.configure();
- CollectionAdminRequest.ClusterProp clusterPropRequest = CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, "false");
+ Boolean useLegacyCloud = rarely();
+ LOG.info("Using legacyCloud?: {}", useLegacyCloud);
+ CollectionAdminRequest.ClusterProp clusterPropRequest = CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, String.valueOf(useLegacyCloud));
CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
assertEquals(0, response.getStatus());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0330b4ab/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
index 80cc25e..e158a92 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
@@ -77,7 +77,9 @@ public class TestPassiveReplica extends SolrCloudTestCase {
configureCluster(2) // 2 + random().nextInt(3)
.addConfig("conf", configset("cloud-minimal"))
.configure();
- CollectionAdminRequest.ClusterProp clusterPropRequest = CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, "false");
+ Boolean useLegacyCloud = rarely();
+ LOG.info("Using legacyCloud?: {}", useLegacyCloud);
+ CollectionAdminRequest.ClusterProp clusterPropRequest = CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, String.valueOf(useLegacyCloud));
CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
assertEquals(0, response.getStatus());
}
@@ -287,10 +289,6 @@ public class TestPassiveReplica extends SolrCloudTestCase {
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), Replica.State.ACTIVE, statesSeen.get(0));
}
- public void testPassiveReplicaCantConnectToZooKeeper() {
-
- }
-
public void testRealTimeGet() {
// should be redirected to writers
}
[2/2] lucene-solr:jira/solr-10233: Added ChaosMonkey tests with safe
leader for passive replicas
Posted by tf...@apache.org.
Added ChaosMonkey tests with safe leader for passive replicas
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/304add6f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/304add6f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/304add6f
Branch: refs/heads/jira/solr-10233
Commit: 304add6f631494d28d952431055e89b8357c6a5a
Parents: 0330b4a
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed Apr 26 16:28:25 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed Apr 26 16:28:25 2017 -0700
----------------------------------------------------------------------
.../src/java/org/apache/solr/cloud/Assign.java | 1 +
.../org/apache/solr/cloud/CloudDescriptor.java | 3 +-
.../apache/solr/cloud/CreateCollectionCmd.java | 2 -
solr/core/src/test-files/log4j.properties | 2 +
...MonkeySafeLeaderWithPassiveReplicasTest.java | 229 +++++++++++++++++++
.../cloud/TestPassiveReplicaErrorHandling.java | 2 -
.../solr/BaseDistributedSearchTestCase.java | 1 +
.../cloud/AbstractFullDistribZkTestBase.java | 63 +++--
8 files changed, 284 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/core/src/java/org/apache/solr/cloud/Assign.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Assign.java b/solr/core/src/java/org/apache/solr/cloud/Assign.java
index ca784e5..924ff17 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Assign.java
@@ -109,6 +109,7 @@ public class Assign {
}
public static String buildCoreName(String collectionName, String shard, Replica.Type type, int replicaNum) {
+ // TODO: Adding the suffix is great for debugging, but may be an issue if at some point we want to support a way to change replica type
return collectionName + "_" + shard + "_replica_" + type.name().substring(0,1).toLowerCase() + replicaNum;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java b/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
index ff29afc..c39272c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
@@ -64,7 +64,8 @@ public class CloudDescriptor {
if (Strings.isNullOrEmpty(nodeName))
this.nodeName = null;
this.numShards = PropertiesUtil.toInteger(props.getProperty(CloudDescriptor.NUM_SHARDS), null);
- this.replicaType = Replica.Type.valueOf(props.getProperty(CloudDescriptor.REPLICA_TYPE, Replica.Type.REALTIME.name()));
+ System.out.println("ReplicaType: " + props);
+ this.replicaType = Replica.Type.valueOf(props.getProperty(CloudDescriptor.REPLICA_TYPE, Replica.Type.REALTIME.toString()));
for (String propName : props.stringPropertyNames()) {
if (propName.startsWith(ZkController.COLLECTION_PARAM_PREFIX)) {
collectionParams.put(propName.substring(ZkController.COLLECTION_PARAM_PREFIX.length()), props.getProperty(propName));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
index 41b8420..4b6971e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
@@ -209,8 +209,6 @@ public class CreateCollectionCmd implements Cmd {
for (Map.Entry<ReplicaAssigner.Position, String> e : positionVsNodes.entrySet()) {
ReplicaAssigner.Position position = e.getKey();
String nodeName = e.getValue();
- // TODO: Adding the suffix is great for debugging, but may be an issue if at some point we want to support a way to change replica type
-// String coreName = collectionName + "_" + position.shard + "_replica" + position.suffix + (position.index + 1);
String coreName = Assign.buildCoreName(collectionName, position.shard, position.type, position.index + 1);
log.debug(formatString("Creating core {0} as part of shard {1} of collection {2} on {3}"
, coreName, position.shard, collectionName, nodeName));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/core/src/test-files/log4j.properties
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/log4j.properties b/solr/core/src/test-files/log4j.properties
index c464a9f..73f7553 100644
--- a/solr/core/src/test-files/log4j.properties
+++ b/solr/core/src/test-files/log4j.properties
@@ -1,4 +1,5 @@
# Logging level
+# nocommit: revert this file before back to master
log4j.rootLogger=DEBUG, CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
@@ -16,6 +17,7 @@ log4j.logger.org.apache.solr.cloud.OverseerTaskQueue=INFO
log4j.logger.org.apache.solr.common.cloud.SolrZkClient=INFO
log4j.logger.org.apache.solr.util.stats.InstrumentedPoolingHttpClientConnectionManager=INFO
log4j.logger.com.codehale.metrics=INFO
+log4j.logger.com.codahale.metrics.JmxReporter=INFO
#log4j.logger.org.apache.solr.update.processor.LogUpdateProcessorFactory=DEBUG
#log4j.logger.org.apache.solr.update.processor.DistributedUpdateProcessor=DEBUG
#log4j.logger.org.apache.solr.update.PeerSync=DEBUG
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
new file mode 100644
index 0000000..5fee604
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Slow
+public class ChaosMonkeySafeLeaderWithPassiveReplicasTest extends AbstractFullDistribZkTestBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
+
+ private final int numPassiveReplicas;
+ private final int numRealtimeOrAppendReplicas;
+
+ protected int getPassiveReplicaCount() {
+ return numPassiveReplicas;
+ }
+
+ @BeforeClass
+ public static void beforeSuperClass() {
+ schemaString = "schema15.xml"; // we need a string id
+ System.setProperty("solr.autoCommit.maxTime", "15000");
+ setErrorHook();
+ }
+
+ @AfterClass
+ public static void afterSuperClass() {
+ System.clearProperty("solr.autoCommit.maxTime");
+ clearErrorHook();
+ }
+
+ protected static final String[] fieldNames = new String[]{"f_i", "f_f", "f_d", "f_l", "f_dt"};
+ protected static final RandVal[] randVals = new RandVal[]{rint, rfloat, rdouble, rlong, rdate};
+
+ public String[] getFieldNames() {
+ return fieldNames;
+ }
+
+ public RandVal[] getRandValues() {
+ return randVals;
+ }
+
+ @Override
+ public void distribSetUp() throws Exception {
+ useFactory("solr.StandardDirectoryFactory");
+ super.distribSetUp();
+ }
+
+ public ChaosMonkeySafeLeaderWithPassiveReplicasTest() {
+ super();
+ numPassiveReplicas = random().nextInt(TEST_NIGHTLY ? 3 : 2) + 1;;
+ numRealtimeOrAppendReplicas = random().nextInt(TEST_NIGHTLY ? 3 : 2) + 1;;
+ sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1"));
+ if (sliceCount == -1) {
+ sliceCount = random().nextInt(TEST_NIGHTLY ? 3 : 2) + 1;
+ }
+
+ int numNodes = sliceCount * (numRealtimeOrAppendReplicas + numPassiveReplicas);
+ fixShardCount(numNodes);
+ log.info("Starting ChaosMonkey test with {} shards and {} nodes", sliceCount, numNodes);
+ }
+
+ @Test
+ public void test() throws Exception {
+ DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION);
+ assertEquals(this.sliceCount, docCollection.getSlices().size());
+ Slice s = docCollection.getSlice("shard1");
+ assertNotNull(s);
+ assertEquals("Unexpected number of replicas. Collection: " + docCollection, numRealtimeOrAppendReplicas + numPassiveReplicas, s.getReplicas().size());
+ assertEquals("Unexpected number of passive replicas. Collection: " + docCollection, numPassiveReplicas, s.getReplicas(EnumSet.of(Replica.Type.PASSIVE)).size());
+ assertEquals(useAppendReplicas()?0:numRealtimeOrAppendReplicas, s.getReplicas(EnumSet.of(Replica.Type.REALTIME)).size());
+ assertEquals(useAppendReplicas()?numRealtimeOrAppendReplicas:0, s.getReplicas(EnumSet.of(Replica.Type.APPEND)).size());
+ handle.clear();
+ handle.put("timestamp", SKIPVAL);
+
+ // randomly turn on 1 seconds 'soft' commit
+ randomlyEnableAutoSoftCommit();
+
+ tryDelete();
+
+ List<StoppableIndexingThread> threads = new ArrayList<>();
+ int threadCount = 2;
+ int batchSize = 1;
+ if (random().nextBoolean()) {
+ batchSize = random().nextInt(98) + 2;
+ }
+
+ boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true;
+ int maxUpdates = -1;
+ if (!pauseBetweenUpdates) {
+ maxUpdates = 1000 + random().nextInt(1000);
+ } else {
+ maxUpdates = 15000;
+ }
+
+ for (int i = 0; i < threadCount; i++) {
+ StoppableIndexingThread indexThread = new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true, maxUpdates, batchSize, pauseBetweenUpdates); // random().nextInt(999) + 1
+ threads.add(indexThread);
+ indexThread.start();
+ }
+
+ chaosMonkey.startTheMonkey(false, 500);
+ try {
+ long runLength;
+ if (RUN_LENGTH != -1) {
+ runLength = RUN_LENGTH;
+ } else {
+ int[] runTimes;
+ if (TEST_NIGHTLY) {
+ runTimes = new int[] {5000, 6000, 10000, 15000, 25000, 30000,
+ 30000, 45000, 90000, 120000};
+ } else {
+ runTimes = new int[] {5000, 7000, 15000};
+ }
+ runLength = runTimes[random().nextInt(runTimes.length - 1)];
+ }
+
+ Thread.sleep(runLength);
+ } finally {
+ chaosMonkey.stopTheMonkey();
+ }
+
+ for (StoppableIndexingThread indexThread : threads) {
+ indexThread.safeStop();
+ }
+
+ // wait for stop...
+ for (StoppableIndexingThread thread : threads) {
+ thread.join();
+ }
+
+ for (StoppableIndexingThread indexThread : threads) {
+ assertEquals(0, indexThread.getFailCount());
+ }
+
+ // try and wait for any replications and what not to finish...
+
+ Thread.sleep(2000);
+
+ waitForThingsToLevelOut(180000);
+
+ // even if things were leveled out, a jetty may have just been stopped or something
+ // we wait again and wait to level out again to make sure the system is not still in flux
+
+ Thread.sleep(3000);
+
+ waitForThingsToLevelOut(180000);
+
+ checkShardConsistency(batchSize == 1, true);
+
+ log.info("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
+
+ // try and make a collection to make sure the overseer has survived the expiration and session loss
+
+ // sometimes we restart zookeeper as well
+ if (random().nextBoolean()) {
+ zkServer.shutdown();
+ zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
+ zkServer.run();
+ }
+
+ try (CloudSolrClient client = createCloudClient("collection1")) {
+ createCollection(null, "testcollection", 1, 1, 100, client, null, "conf1");
+
+ }
+ List<Integer> numShardsNumReplicas = new ArrayList<>(2);
+ numShardsNumReplicas.add(1);
+ numShardsNumReplicas.add(1 + getPassiveReplicaCount());
+ checkForCollection("testcollection",numShardsNumReplicas, null);
+ }
+
+ private void tryDelete() throws Exception {
+ long start = System.nanoTime();
+ long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);
+ while (System.nanoTime() < timeout) {
+ try {
+ del("*:*");
+ break;
+ } catch (SolrServerException e) {
+ // cluster may not be up yet
+ e.printStackTrace();
+ }
+ Thread.sleep(100);
+ }
+ }
+
+ // skip the randoms - they can deadlock...
+ @Override
+ protected void indexr(Object... fields) throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, fields);
+ addFields(doc, "rnd_b", true);
+ indexDoc(doc);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
index cbc3f2a..7dd147a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
@@ -103,8 +103,6 @@ public class TestPassiveReplicaErrorHandling extends SolrCloudTestCase {
@AfterClass
public static void tearDownCluster() throws Exception {
-// cluster.shutdown();
-// cluster = null;
for (SocketProxy proxy:proxies.values()) {
proxy.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
index 8c6eb60..9213fee 100644
--- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
@@ -1123,6 +1123,7 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
coreProperties.setProperty("config", "${solrconfig:solrconfig.xml}");
coreProperties.setProperty("schema", "${schema:schema.xml}");
coreProperties.setProperty("coreNodeName", "${coreNodeName:}");
+ coreProperties.setProperty("replicaType", "${replicaType:}");
writeCoreProperties(jettyHome.toPath().resolve("cores").resolve("collection1"), coreProperties, "collection1");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/304add6f/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index e9e0f3d..acd425b 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -379,17 +379,22 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
StringBuilder sb = new StringBuilder();
if ("2".equals(getStateFormat())) {
- log.info("Creating collection1 with stateFormat=2");
+ log.info("Creating " + DEFAULT_COLLECTION + " with stateFormat=2");
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(),
AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT);
Overseer.getStateUpdateQueue(zkClient).offer(
Utils.toJSON(Utils.makeMap(Overseer.QUEUE_OPERATION,
- CollectionParams.CollectionAction.CREATE.toLower(), "name",
- DEFAULT_COLLECTION, "numShards", String.valueOf(sliceCount),
+ CollectionParams.CollectionAction.CREATE.toLower(),
+ "name", DEFAULT_COLLECTION,
+ "numShards", String.valueOf(sliceCount),
DocCollection.STATE_FORMAT, getStateFormat(),
- ZkStateReader.REALTIME_REPLICAS, useAppendReplicas())));
+ ZkStateReader.REALTIME_REPLICAS, useAppendReplicas()?"0":"1",
+ ZkStateReader.APPEND_REPLICAS, useAppendReplicas()?"1":"0",
+ ZkStateReader.PASSIVE_REPLICAS, String.valueOf(getPassiveReplicaCount()))));
zkClient.close();
}
+
+ int numPassiveReplicas = getPassiveReplicaCount() * sliceCount;
for (int i = 1; i <= numJettys; i++) {
if (sb.length() > 0) sb.append(',');
@@ -399,9 +404,22 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
jettyDir.mkdirs();
setupJettySolrHome(jettyDir);
- log.info("create jetty {} in directory {}", i, jettyDir);
- JettySolrRunner j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty"
- + cnt) : null, null, "solrconfig.xml", null);
+ JettySolrRunner j;
+
+ if (numPassiveReplicas > 0) {
+ numPassiveReplicas--;
+ log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.PASSIVE);
+ j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty"
+ + cnt) : null, null, "solrconfig.xml", null, Replica.Type.PASSIVE);
+ } else if (useAppendReplicas()) {
+ log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.APPEND);
+ j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty"
+ + cnt) : null, null, "solrconfig.xml", null, Replica.Type.APPEND);
+ } else {
+ log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.REALTIME);
+ j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty"
+ + cnt) : null, null, "solrconfig.xml", null, null);
+ }
jettys.add(j);
SolrClient client = createNewSolrClient(j.getLocalPort());
clients.add(client);
@@ -410,17 +428,18 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
this.jettys.addAll(jettys);
this.clients.addAll(clients);
- int numShards = getTotalReplicas(DEFAULT_COLLECTION);
+ int numReplicas = getTotalReplicas(DEFAULT_COLLECTION);
+ int expectedNumReplicas = numJettys;
// now wait until we see that the number of shards in the cluster state
// matches what we expect
int retries = 0;
- while (numShards != getShardCount()) {
- numShards = getTotalReplicas(DEFAULT_COLLECTION);
- if (numShards == getShardCount()) break;
+ while (numReplicas != expectedNumReplicas) {
+ numReplicas = getTotalReplicas(DEFAULT_COLLECTION);
+ if (numReplicas == expectedNumReplicas) break;
if (retries++ == 60) {
printLayoutOnTearDown = true;
- fail("Shards in the state does not match what we set:" + numShards + " vs " + getShardCount());
+ fail("Number of replicas in the state does not match what we set:" + numReplicas + " vs " + expectedNumReplicas);
}
Thread.sleep(500);
}
@@ -431,7 +450,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + i, 10000);
}
- if (numShards > 0) {
+ if (numReplicas > 0) {
updateMappingsFromZk(this.jettys, this.clients);
}
@@ -449,6 +468,10 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
}
+ protected int getPassiveReplicaCount() {
+ return 0;
+ }
+
/* Total number of replicas (number of cores serving an index to the collection) shown by the cluster state */
protected int getTotalReplicas(String collection) {
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
@@ -484,8 +507,12 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
return jetty;
}
-
+
public JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, String solrConfigOverride, String schemaOverride) throws Exception {
+ return createJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, null);
+ }
+
+ public JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType) throws Exception {
// randomly test a relative solr.home path
if (random().nextBoolean()) {
solrHome = getRelativeSolrHomePath(solrHome);
@@ -508,6 +535,11 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
props.setProperty("shards", shardList);
if (dataDir != null)
props.setProperty("solr.data.dir", getDataDir(dataDir));
+ if (replicaType != null) {
+ props.setProperty("replicaType", replicaType.toString());
+ } else { // TODO: include the case with no replicaTYpe defined: if (random().nextBoolean()) {
+ props.setProperty("replicaType", Replica.Type.REALTIME.toString());
+ }
props.setProperty("coreRootDirectory", solrHome.toPath().resolve("cores").toAbsolutePath().toString());
JettySolrRunner jetty = new JettySolrRunner(solrHome.getPath(), props, jettyconfig);
@@ -1630,6 +1662,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
NUM_SLICES, numShards,
ZkStateReader.REALTIME_REPLICAS, numRealtimeReplicas,
ZkStateReader.APPEND_REPLICAS, numAppendReplicas,
+ ZkStateReader.PASSIVE_REPLICAS, getPassiveReplicaCount(),
CREATE_NODE_SET, createNodeSetStr,
ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode),
client);
@@ -1645,6 +1678,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
NUM_SLICES, numShards,
ZkStateReader.REALTIME_REPLICAS, numRealtimeReplicas,
ZkStateReader.APPEND_REPLICAS, numAppendReplicas,
+ ZkStateReader.PASSIVE_REPLICAS, getPassiveReplicaCount(),
CREATE_NODE_SET, createNodeSetStr,
ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode),
client, configName);
@@ -1831,6 +1865,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode,
ZkStateReader.REALTIME_REPLICAS, numRealtimeReplicas,
ZkStateReader.APPEND_REPLICAS, numAppendReplicas,
+ ZkStateReader.PASSIVE_REPLICAS, getPassiveReplicaCount(),
NUM_SLICES, numShards);
Map<String,List<Integer>> collectionInfos = new HashMap<>();
createCollection(collectionInfos, collName, props, client);