You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2017/05/10 22:09:55 UTC
[1/6] lucene-solr:jira/solr-10233: Added support for replica types in
create shard operation
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-10233 b16f0e353 -> 71e1149d4
Added support for replica types in create shard operation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d78946da
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d78946da
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d78946da
Branch: refs/heads/jira/solr-10233
Commit: d78946daf122aeb740edc0182ca0fa74d146c3d6
Parents: b16f0e3
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Tue May 9 13:03:30 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Tue May 9 13:03:30 2017 -0700
----------------------------------------------------------------------
.../org/apache/solr/cloud/CreateShardCmd.java | 48 ++++++++++++++++----
.../solr/cloud/CollectionsAPISolrJTest.java | 12 +++--
.../solrj/request/CollectionAdminRequest.java | 23 +++++++---
.../apache/solr/common/cloud/ZkStateReader.java | 4 --
4 files changed, 64 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d78946da/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
index 52df32b..f38009c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateShardCmd.java
@@ -28,6 +28,7 @@ import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
@@ -41,7 +42,10 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.cloud.Assign.getNodesForNewReplicas;
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF;
+import static org.apache.solr.common.cloud.ZkStateReader.APPEND_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.PASSIVE_REPLICAS;
+import static org.apache.solr.common.cloud.ZkStateReader.REALTIME_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@@ -67,9 +71,18 @@ public class CreateShardCmd implements Cmd {
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
DocCollection collection = clusterState.getCollection(collectionName);
- int repFactor = message.getInt(REPLICATION_FACTOR, collection.getInt(REPLICATION_FACTOR, 1));
+// int repFactor = message.getInt(REPLICATION_FACTOR, collection.getInt(REPLICATION_FACTOR, 1));
+ int numRealtimeReplicas = message.getInt(REALTIME_REPLICAS, message.getInt(REPLICATION_FACTOR, collection.getInt(REALTIME_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1))));
+ int numPassiveReplicas = message.getInt(PASSIVE_REPLICAS, collection.getInt(PASSIVE_REPLICAS, 0));
+ int numAppendReplicas = message.getInt(APPEND_REPLICAS, collection.getInt(APPEND_REPLICAS, 0));
+ int totalReplicas = numRealtimeReplicas + numPassiveReplicas + numAppendReplicas;
+
+ if (numRealtimeReplicas + numAppendReplicas <= 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, REALTIME_REPLICAS + " + " + APPEND_REPLICAS + " must be greater than 0");
+ }
+
Object createNodeSetStr = message.get(OverseerCollectionMessageHandler.CREATE_NODE_SET);
- List<Assign.ReplicaCount> sortedNodeList = getNodesForNewReplicas(clusterState, collectionName, sliceName, repFactor,
+ List<Assign.ReplicaCount> sortedNodeList = getNodesForNewReplicas(clusterState, collectionName, sliceName, totalReplicas,
createNodeSetStr, ocmh.overseer.getZkController().getCoreContainer());
ZkStateReader zkStateReader = ocmh.zkStateReader;
@@ -90,19 +103,38 @@ public class CreateShardCmd implements Cmd {
String async = message.getStr(ASYNC);
Map<String, String> requestMap = null;
if (async != null) {
- requestMap = new HashMap<>(repFactor, 1.0f);
+ requestMap = new HashMap<>(totalReplicas, 1.0f);
}
-
- for (int j = 1; j <= repFactor; j++) {
+
+ int createdRealtimeReplicas = 0, createdAppendReplicas = 0, createdPassiveReplicas = 0;
+
+ for (int j = 1; j <= totalReplicas; j++) {
+ int coreNameNumber;
+ Replica.Type typeToCreate;
+ if (createdRealtimeReplicas < numRealtimeReplicas) {
+ createdRealtimeReplicas++;
+ coreNameNumber = createdRealtimeReplicas;
+ typeToCreate = Replica.Type.REALTIME;
+ } else if (createdAppendReplicas < numAppendReplicas) {
+ createdAppendReplicas++;
+ coreNameNumber = createdAppendReplicas;
+ typeToCreate = Replica.Type.APPEND;
+ } else {
+ createdPassiveReplicas++;
+ coreNameNumber = createdPassiveReplicas;
+ typeToCreate = Replica.Type.PASSIVE;
+ }
String nodeName = sortedNodeList.get(((j - 1)) % sortedNodeList.size()).nodeName;
- String shardName = collectionName + "_" + sliceName + "_replica" + j;
- log.info("Creating shard " + shardName + " as part of slice " + sliceName + " of collection " + collectionName
+ String coreName = Assign.buildCoreName(collectionName, sliceName, typeToCreate, coreNameNumber);
+// String coreName = collectionName + "_" + sliceName + "_replica" + j;
+ log.info("Creating replica " + coreName + " as part of slice " + sliceName + " of collection " + collectionName
+ " on " + nodeName);
// Need to create new params for each request
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString());
- params.set(CoreAdminParams.NAME, shardName);
+ params.set(CoreAdminParams.NAME, coreName);
+ params.set(CoreAdminParams.REPLICA_TYPE, typeToCreate.name());
params.set(COLL_CONF, configName);
params.set(CoreAdminParams.COLLECTION, collectionName);
params.set(CoreAdminParams.SHARD, sliceName);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d78946da/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 93f61b5..2a2da78 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -117,17 +117,17 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
@Test
public void testCreateAndDeleteShard() throws IOException, SolrServerException {
-
// Create an implicit collection
String collectionName = "solrj_implicit";
CollectionAdminResponse response
- = CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "shardA,shardB", 1)
+ = CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "shardA,shardB", 1, 1, 1)
+ .setMaxShardsPerNode(3)
.process(cluster.getSolrClient());
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
Map<String, NamedList<Integer>> coresStatus = response.getCollectionCoresStatus();
- assertEquals(2, coresStatus.size());
+ assertEquals(6, coresStatus.size());
// Add a shard to the implicit collection
response = CollectionAdminRequest.createShard(collectionName, "shardC").process(cluster.getSolrClient());
@@ -135,8 +135,10 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
coresStatus = response.getCollectionCoresStatus();
- assertEquals(1, coresStatus.size());
- assertEquals(0, (int) coresStatus.get(collectionName + "_shardC_replica1").get("status"));
+ assertEquals(3, coresStatus.size());
+ assertEquals(0, (int) coresStatus.get(Assign.buildCoreName(collectionName, "shardC", Replica.Type.REALTIME, 1)).get("status"));
+ assertEquals(0, (int) coresStatus.get(Assign.buildCoreName(collectionName, "shardC", Replica.Type.APPEND, 1)).get("status"));
+ assertEquals(0, (int) coresStatus.get(Assign.buildCoreName(collectionName, "shardC", Replica.Type.PASSIVE, 1)).get("status"));
response = CollectionAdminRequest.deleteShard(collectionName, "shardC").process(cluster.getSolrClient());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d78946da/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index b248f92..10b5c69 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -365,6 +365,22 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public static Create createCollectionWithImplicitRouter(String collection, String config, String shards, int numReplicas) {
return new Create(collection, config, shards, numReplicas);
}
+
+ /**
+ * Returns a SolrRequest for creating a collection with the implicit router and specific types of replicas
+ * @param collection the collection name
+ * @param config the collection config
+ * @param shards a shard definition string
+ * @param numRealtimeReplicas the number of replicas of type {@link org.apache.solr.common.cloud.Replica.Type#REALTIME}
+ * @param numAppendReplicas the number of replicas of type {@link org.apache.solr.common.cloud.Replica.Type#APPEND}
+ * @param numPassiveReplicas the number of replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PASSIVE}
+ */
+ public static Create createCollectionWithImplicitRouter(String collection, String config, String shards, int numRealtimeReplicas, int numAppendReplicas, int numPassiveReplicas) {
+ Create createRequest = new Create(collection, config, shards, numRealtimeReplicas);
+ createRequest.appendReplicas = numAppendReplicas;
+ createRequest.passiveReplicas = numPassiveReplicas;
+ return createRequest;
+ }
// CREATE request
public static class Create extends AsyncCollectionSpecificAdminRequest {
@@ -422,7 +438,6 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public Create setRealtimeReplicas(Integer realtimeReplicas) { this.realtimeReplicas = realtimeReplicas; return this;}
public Create setAppendReplicas(Integer appendReplicas) { this.appendReplicas = appendReplicas; return this;}
- @Deprecated
public Create setReplicationFactor(Integer repl) { this.realtimeReplicas = repl; return this; }
public Create setStateFormat(Integer stateFormat) { this.stateFormat = stateFormat; return this; }
public Create setRule(String... s){ this.rule = s; return this; }
@@ -434,11 +449,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public String getShards() { return shards; }
public Integer getNumShards() { return numShards; }
public Integer getMaxShardsPerNode() { return maxShardsPerNode; }
- /**
- *
- * @deprecated Use {@link #getNumRealtimeReplicas()}
- */
- @Deprecated
+
public Integer getReplicationFactor() { return getNumRealtimeReplicas(); }
public Integer getNumRealtimeReplicas() { return realtimeReplicas; }
public Boolean getAutoAddReplicas() { return autoAddReplicas; }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d78946da/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index c0a6caf..3f6f1ac 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -94,10 +94,6 @@ public class ZkStateReader implements Closeable {
public static final String REJOIN_AT_HEAD_PROP = "rejoinAtHead";
public static final String SOLR_SECURITY_CONF_PATH = "/security.json";
- /**
- *@deprecated Use {@link #REALTIME_REPLICAS}
- */
- @Deprecated
public static final String REPLICATION_FACTOR = "replicationFactor";
public static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
public static final String AUTO_ADD_REPLICAS = "autoAddReplicas";
[6/6] lucene-solr:jira/solr-10233: Reset TestInjector after modifying
it in tests
Posted by tf...@apache.org.
Reset TestInjector after modifying it in tests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/71e1149d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/71e1149d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/71e1149d
Branch: refs/heads/jira/solr-10233
Commit: 71e1149d40bbc700aef34d98907142ef38e8dbb8
Parents: 1c53272
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed May 10 15:09:45 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed May 10 15:09:45 2017 -0700
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/util/TestInjection.java | 6 ++++--
.../ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java | 1 +
.../cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java | 1 +
.../src/test/org/apache/solr/cloud/TestAppendReplica.java | 7 ++++++-
.../src/test/org/apache/solr/cloud/TestPassiveReplica.java | 6 ++++++
.../apache/solr/cloud/TestPassiveReplicaErrorHandling.java | 1 +
6 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71e1149d/solr/core/src/java/org/apache/solr/util/TestInjection.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java
index 6b7b1f8..f3eb484 100644
--- a/solr/core/src/java/org/apache/solr/util/TestInjection.java
+++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java
@@ -75,7 +75,7 @@ public class TestInjection {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final Pattern ENABLED_PERCENT = Pattern.compile("(true|false)(?:\\:(\\d+))?$", Pattern.CASE_INSENSITIVE);
-
+
private static final String LUCENE_TEST_CASE_FQN = "org.apache.lucene.util.LuceneTestCase";
/**
@@ -151,6 +151,7 @@ public class TestInjection {
splitFailureBeforeReplicaCreation = null;
prepRecoveryOpPauseForever = null;
countPrepRecoveryOpPauseForever = new AtomicInteger(0);
+ waitForReplicasInSync = "true:60";
for (Timer timer : timers) {
timer.cancel();
@@ -387,9 +388,10 @@ public class TestInjection {
String localVersion = searcher.get().getIndexReader().getIndexCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
if (localVersion == null && leaderVersion == 0 && !core.getUpdateHandler().getUpdateLog().hasUncommittedChanges()) return true;
if (localVersion != null && Long.parseLong(localVersion) == leaderVersion && (leaderVersion >= t || i >= 6)) {
- log.info("Waiting time for replica in sync with leader: {}", System.currentTimeMillis()-currentTime);
+ log.info("Waiting time for append replica to be in sync with leader: {}", System.currentTimeMillis()-currentTime);
return true;
} else {
+ log.debug("Append replica not in sync with leader yet. Attempt: {}", i);
Thread.sleep(500);
}
} finally {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71e1149d/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
index ce35529..94d6e45 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeWithPassiveReplicasTest.java
@@ -78,6 +78,7 @@ public class ChaosMonkeyNothingIsSafeWithPassiveReplicasTest extends AbstractFul
public static void afterSuperClass() {
System.clearProperty("solr.autoCommit.maxTime");
clearErrorHook();
+ TestInjection.reset();
}
protected static final String[] fieldNames = new String[]{"f_i", "f_f", "f_d", "f_l", "f_dt"};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71e1149d/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
index 556a61b..8639fba 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderWithPassiveReplicasTest.java
@@ -73,6 +73,7 @@ public class ChaosMonkeySafeLeaderWithPassiveReplicasTest extends AbstractFullDi
public static void afterSuperClass() {
System.clearProperty("solr.autoCommit.maxTime");
clearErrorHook();
+ TestInjection.reset();
}
protected static final String[] fieldNames = new String[]{"f_i", "f_f", "f_d", "f_l", "f_dt"};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71e1149d/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
index 23dde01..a0cc1f5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
@@ -61,8 +61,8 @@ import org.apache.solr.util.RefCounted;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,6 +93,11 @@ public class TestAppendReplica extends SolrCloudTestCase {
assertEquals(0, response.getStatus());
}
+ @AfterClass
+ public static void tearDownCluster() {
+ TestInjection.reset();
+ }
+
@Override
public void setUp() throws Exception {
super.setUp();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71e1149d/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
index 70f2627..c9cacbb 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
@@ -51,6 +51,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.slf4j.Logger;
@@ -83,6 +84,11 @@ public class TestPassiveReplica extends SolrCloudTestCase {
assertEquals(0, response.getStatus());
}
+ @AfterClass
+ public static void tearDownCluster() {
+ TestInjection.reset();
+ }
+
@Override
public void setUp() throws Exception {
super.setUp();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71e1149d/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
index 7dd147a..f331ef6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplicaErrorHandling.java
@@ -108,6 +108,7 @@ public class TestPassiveReplicaErrorHandling extends SolrCloudTestCase {
}
proxies = null;
jettys = null;
+ TestInjection.reset();
}
@Override
[2/6] lucene-solr:jira/solr-10233: Minor improvements to recovery code
Posted by tf...@apache.org.
Minor improvements to recovery code
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/507548a9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/507548a9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/507548a9
Branch: refs/heads/jira/solr-10233
Commit: 507548a94e1f8e95e76f3a1d6a41a1d5b29210e5
Parents: d78946d
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Tue May 9 13:04:59 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Tue May 9 13:04:59 2017 -0700
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java | 6 +-----
solr/core/src/java/org/apache/solr/cloud/ZkController.java | 3 +++
.../src/java/org/apache/solr/handler/ReplicationHandler.java | 4 ++++
.../src/java/org/apache/solr/handler/admin/PrepRecoveryOp.java | 2 +-
4 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/507548a9/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
index 53b6f3e..3c8c1f4 100644
--- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
+++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
@@ -330,6 +330,7 @@ public class RecoveryStrategy implements Runnable, Closeable {
throw new SolrException(ErrorCode.SERVER_ERROR, "Cloud state still says we are leader.");
}
if (cloudDesc.isLeader()) {
+ assert cloudDesc.getReplicaType() != Replica.Type.PASSIVE;
// we are now the leader - no one else must have been suitable
LOG.warn("We have not yet recovered - but we are now the leader!");
LOG.info("Finished recovery process.");
@@ -346,11 +347,6 @@ public class RecoveryStrategy implements Runnable, Closeable {
LOG.info("Recovery for core {} has been closed", core.getName());
break;
}
-
- if (isClosed()) {
- LOG.info("Recovery for core {} has been closed", core.getName());
- break;
- }
LOG.info("Starting Replication Recovery.");
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/507548a9/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 5fdd92c..11fff1f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -892,6 +892,9 @@ public class ZkController {
if (replica == null || replica.getType() != Type.PASSIVE) {
joinElection(desc, afterExpiration, joinAtHead);
} else if (replica.getType() == Type.PASSIVE) {
+ if (joinAtHead) {
+ log.warn("Replica {} was designated as preferred leader but it's type is {}, It won't join election", coreZkNodeName, Type.PASSIVE);
+ }
log.debug("Replica {} skipping election because it's type is {}", coreZkNodeName, Type.PASSIVE);
startReplicationFromLeader(coreName, false);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/507548a9/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index 94ff189..90d33be 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -403,6 +403,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
String masterUrl = solrParams == null ? null : solrParams.get(MASTER_URL);
if (!indexFetchLock.tryLock())
return IndexFetchResult.LOCK_OBTAIN_FAILED;
+ if (core.getCoreContainer().isShutDown()) {
+ LOG.warn("I was asked to replicate but CoreContainer is shutting down");
+ return IndexFetchResult.LOCK_OBTAIN_FAILED;//nocommit: different
+ }
try {
if (masterUrl != null) {
if (currentIndexFetcher != null && currentIndexFetcher != pollingIndexFetcher) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/507548a9/solr/core/src/java/org/apache/solr/handler/admin/PrepRecoveryOp.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/PrepRecoveryOp.java b/solr/core/src/java/org/apache/solr/handler/admin/PrepRecoveryOp.java
index 8e5408b..39892b0 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/PrepRecoveryOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/PrepRecoveryOp.java
@@ -186,7 +186,7 @@ class PrepRecoveryOp implements CoreAdminHandler.CoreAdminOp {
}
// solrcloud_debug
- if (log.isDebugEnabled()) {
+ if (log.isDebugEnabled() && core != null) {
try {
LocalSolrQueryRequest r = new LocalSolrQueryRequest(core,
new ModifiableSolrParams());
[5/6] lucene-solr:jira/solr-10233: Removed OnlyLeaderIndexesTest.java
(test is TestAppendReplica now) and back to using UpdateHandler stats in
TestAppend and TestPassive
Posted by tf...@apache.org.
Removed OnlyLeaderIndexesTest.java (test is TestAppendReplica now) and back to using UpdateHandler stats in TestAppend and TestPassive
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1c53272e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1c53272e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1c53272e
Branch: refs/heads/jira/solr-10233
Commit: 1c53272e947801bdc1f54ab9ef89fe1e98edf432
Parents: 6271229
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed May 10 13:17:02 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed May 10 13:17:02 2017 -0700
----------------------------------------------------------------------
.../solr/cloud/OnlyLeaderIndexesTest.java | 488 -------------------
.../apache/solr/cloud/TestAppendReplica.java | 30 +-
.../apache/solr/cloud/TestPassiveReplica.java | 7 +-
3 files changed, 6 insertions(+), 519 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1c53272e/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java b/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
deleted file mode 100644
index 6297408..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.concurrent.Semaphore;
-
-import org.apache.lucene.index.IndexWriter;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.CollectionStatePredicate;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.update.DirectUpdateHandler2;
-import org.apache.solr.update.SolrIndexWriter;
-import org.apache.solr.update.UpdateHandler;
-import org.apache.solr.update.UpdateLog;
-import org.apache.solr.util.RefCounted;
-import org.apache.zookeeper.KeeperException;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class OnlyLeaderIndexesTest extends SolrCloudTestCase {
- private static final String COLLECTION = "collection1";
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
- System.setProperty("solr.ulog.numRecordsToKeep", "1000");
-
- configureCluster(3)
- .addConfig("config", TEST_PATH().resolve("configsets")
- .resolve("cloud-minimal-inplace-updates").resolve("conf"))
- .configure();
-
- CollectionAdminRequest.ClusterProp clusterPropRequest = CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, "false");
- CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
- assertEquals(0, response.getStatus());
-
- CollectionAdminRequest
- .createCollection(COLLECTION, "config", 1, 0, 3, 0)
- .setMaxShardsPerNode(1)
- .process(cluster.getSolrClient());
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 30);
- }
-
- @Test
- public void test() throws Exception {
- assertNumberOfReplicas(0, 3, 0, false, true);
- basicTest();
- recoveryTest();
- dbiTest();
- basicLeaderElectionTest();
- outOfOrderDBQWithInPlaceUpdatesTest();
- }
-
- public void basicTest() throws Exception {
- CloudSolrClient cloudClient = cluster.getSolrClient();
- new UpdateRequest()
- .add(sdoc("id", "1"))
- .add(sdoc("id", "2"))
- .add(sdoc("id", "3"))
- .add(sdoc("id", "4"))
- .process(cloudClient, COLLECTION);
-
- {
- UpdateHandler updateHandler = getSolrCore(true).get(0).getUpdateHandler();
- RefCounted<IndexWriter> iwRef = updateHandler.getSolrCoreState().getIndexWriter(null);
- assertTrue("IndexWriter at leader must see updates ", iwRef.get().hasUncommittedChanges());
- iwRef.decref();
- }
-
- for (SolrCore solrCore : getSolrCore(false)) {
- RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
- assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
- iwRef.decref();
- }
-
- checkRTG(1, 4, cluster.getJettySolrRunners());
-
- new UpdateRequest()
- .deleteById("1")
- .deleteByQuery("id:2")
- .process(cloudClient, COLLECTION);
-
- // The DBQ is not processed at replicas, so we still can get doc2 and other docs by RTG
- checkRTG(2,4, getSolrRunner(false));
-
- new UpdateRequest()
- .commit(cloudClient, COLLECTION);
-
- checkShardConsistency(2, 1);
-
- // Update log roll over
- for (SolrCore solrCore : getSolrCore(false)) {
- UpdateLog updateLog = solrCore.getUpdateHandler().getUpdateLog();
- assertFalse(updateLog.hasUncommittedChanges());
- }
-
- // UpdateLog copy over old updates
- for (int i = 15; i <= 150; i++) {
- cloudClient.add(COLLECTION, sdoc("id",String.valueOf(i)));
- if (random().nextInt(100) < 15 & i != 150) {
- cloudClient.commit(COLLECTION);
- }
- }
- checkRTG(120,150, cluster.getJettySolrRunners());
- waitForReplicasCatchUp(20);
- }
-
- public void recoveryTest() throws Exception {
- CloudSolrClient cloudClient = cluster.getSolrClient();
- new UpdateRequest()
- .deleteByQuery("*:*")
- .commit(cluster.getSolrClient(), COLLECTION);
- new UpdateRequest()
- .add(sdoc("id", "3"))
- .add(sdoc("id", "4"))
- .commit(cloudClient, COLLECTION);
- // Replica recovery
- new UpdateRequest()
- .add(sdoc("id", "5"))
- .process(cloudClient, COLLECTION);
- JettySolrRunner solrRunner = getSolrRunner(false).get(0);
- ChaosMonkey.stop(solrRunner);
- new UpdateRequest()
- .add(sdoc("id", "6"))
- .process(cloudClient, COLLECTION);
- ChaosMonkey.start(solrRunner);
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 30);
- // We skip peerSync, so replica will always trigger commit on leader
- checkShardConsistency(4, 20);
-
- // LTR can be kicked off, so waiting for replicas recovery
- new UpdateRequest()
- .add(sdoc("id", "7"))
- .commit(cloudClient, COLLECTION);
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 30);
- checkShardConsistency(5, 20);
-
- // More Replica recovery testing
- new UpdateRequest()
- .add(sdoc("id", "8"))
- .process(cloudClient, COLLECTION);
- checkRTG(3,8, cluster.getJettySolrRunners());
- DirectUpdateHandler2.commitOnClose = false;
- ChaosMonkey.stop(solrRunner);
- DirectUpdateHandler2.commitOnClose = true;
- ChaosMonkey.start(solrRunner);
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 30);
- checkRTG(3,8, cluster.getJettySolrRunners());
- checkShardConsistency(6, 20);
-
- // Test replica recovery apply buffer updates
- Semaphore waitingForBufferUpdates = new Semaphore(0);
- Semaphore waitingForReplay = new Semaphore(0);
- RecoveryStrategy.testing_beforeReplayBufferingUpdates = () -> {
- try {
- waitingForReplay.release();
- waitingForBufferUpdates.acquire();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- };
- ChaosMonkey.stop(solrRunner);
- ChaosMonkey.start(solrRunner);
- waitingForReplay.acquire();
- new UpdateRequest()
- .add(sdoc("id", "9"))
- .add(sdoc("id", "10"))
- .process(cloudClient, COLLECTION);
- waitingForBufferUpdates.release();
- RecoveryStrategy.testing_beforeReplayBufferingUpdates = null;
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 30);
- checkRTG(3,10, cluster.getJettySolrRunners());
- checkShardConsistency(6, 20);
- for (SolrCore solrCore : getSolrCore(false)) {
- RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
- assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
- iwRef.decref();
- }
- }
-
- public void dbiTest() throws Exception{
- CloudSolrClient cloudClient = cluster.getSolrClient();
- new UpdateRequest()
- .deleteByQuery("*:*")
- .commit(cluster.getSolrClient(), COLLECTION);
- new UpdateRequest()
- .add(sdoc("id", "1"))
- .commit(cloudClient, COLLECTION);
- checkShardConsistency(1, 1);
- new UpdateRequest()
- .deleteById("1")
- .process(cloudClient, COLLECTION);
- try {
- checkRTG(1, 1, cluster.getJettySolrRunners());
- } catch (AssertionError e) {
- return;
- }
- fail("Doc1 is deleted but it's still exist");
- }
-
- public void basicLeaderElectionTest() throws Exception {
- CloudSolrClient cloudClient = cluster.getSolrClient();
- new UpdateRequest()
- .deleteByQuery("*:*")
- .commit(cluster.getSolrClient(), COLLECTION);
- new UpdateRequest()
- .add(sdoc("id", "1"))
- .add(sdoc("id", "2"))
- .process(cloudClient, COLLECTION);
- String oldLeader = getLeader();
- JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
- ChaosMonkey.kill(oldLeaderJetty);
- for (int i = 0; i < 60; i++) { // wait till leader is changed
- if (!oldLeader.equals(getLeader())) {
- break;
- }
- Thread.sleep(100);
- }
- new UpdateRequest()
- .add(sdoc("id", "3"))
- .add(sdoc("id", "4"))
- .process(cloudClient, COLLECTION);
- ChaosMonkey.start(oldLeaderJetty);
- waitForState("Replica not removed", "collection1", activeReplicaCount(0, 3, 0));
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 60);
- checkRTG(1,4, cluster.getJettySolrRunners());
- new UpdateRequest()
- .commit(cloudClient, COLLECTION);
- checkShardConsistency(4,1);
- }
-
- private String getLeader() throws InterruptedException {
- ZkNodeProps props = cluster.getSolrClient().getZkStateReader().getLeaderRetry("collection1", "shard1", 30000);
- return props.getStr(ZkStateReader.NODE_NAME_PROP);
- }
-
- public void outOfOrderDBQWithInPlaceUpdatesTest() throws Exception {
- new UpdateRequest()
- .deleteByQuery("*:*")
- .commit(cluster.getSolrClient(), COLLECTION);
- List<UpdateRequest> updates = new ArrayList<>();
- updates.add(simulatedUpdateRequest(null, "id", 1, "title_s", "title0_new", "inplace_updatable_int", 5, "_version_", Long.MAX_VALUE-100)); // full update
- updates.add(simulatedDBQ("inplace_updatable_int:5", Long.MAX_VALUE-98));
- updates.add(simulatedUpdateRequest(Long.MAX_VALUE-100, "id", 1, "inplace_updatable_int", 6, "_version_", Long.MAX_VALUE-99));
- for (JettySolrRunner solrRunner: getSolrRunner(false)) {
- try (SolrClient client = solrRunner.newClient()) {
- for (UpdateRequest up : updates) {
- up.process(client, COLLECTION);
- }
- }
- }
- JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
- ChaosMonkey.kill(oldLeaderJetty);
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 30);
- ChaosMonkey.start(oldLeaderJetty);
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
- false, true, 30);
- new UpdateRequest()
- .add(sdoc("id", "2"))
- .commit(cluster.getSolrClient(), COLLECTION);
- checkShardConsistency(2,20);
- SolrDocument doc = cluster.getSolrClient().getById(COLLECTION,"1");
- assertNotNull(doc.get("title_s"));
- }
-
- private UpdateRequest simulatedUpdateRequest(Long prevVersion, Object... fields) throws SolrServerException, IOException {
- SolrInputDocument doc = sdoc(fields);
-
- // get baseUrl of the leader
- String baseUrl = getBaseUrl();
-
- UpdateRequest ur = new UpdateRequest();
- ur.add(doc);
- ur.setParam("update.distrib", "FROMLEADER");
- if (prevVersion != null) {
- ur.setParam("distrib.inplace.prevversion", String.valueOf(prevVersion));
- ur.setParam("distrib.inplace.update", "true");
- }
- ur.setParam("distrib.from", baseUrl);
- return ur;
- }
-
- private UpdateRequest simulatedDBQ(String query, long version) throws SolrServerException, IOException {
- String baseUrl = getBaseUrl();
-
- UpdateRequest ur = new UpdateRequest();
- ur.deleteByQuery(query);
- ur.setParam("_version_", ""+version);
- ur.setParam("update.distrib", "FROMLEADER");
- ur.setParam("distrib.from", baseUrl);
- return ur;
- }
-
- private String getBaseUrl() {
- DocCollection collection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION);
- Slice slice = collection.getSlice("shard1");
- return slice.getLeader().getCoreUrl();
- }
-
- private void checkRTG(int from, int to, List<JettySolrRunner> solrRunners) throws Exception{
-
- for (JettySolrRunner solrRunner: solrRunners) {
- try (SolrClient client = solrRunner.newClient()) {
- for (int i = from; i <= to; i++) {
- SolrQuery query = new SolrQuery("*:*");
- query.set("distrib", false);
- query.setRequestHandler("/get");
- query.set("id",i);
- QueryResponse res = client.query(COLLECTION, query);
- assertNotNull("Can not find doc "+ i + " in " + solrRunner.getBaseUrl(),res.getResponse().get("doc"));
- }
- }
- }
-
- }
-
- private void checkShardConsistency(int expected, int numTry) throws Exception{
- String replicaNotInSync = null;
- for (int i = 0; i < numTry; i++) {
- boolean inSync = true;
- for (JettySolrRunner solrRunner: cluster.getJettySolrRunners()) {
- try (SolrClient client = solrRunner.newClient()) {
- SolrQuery query = new SolrQuery("*:*");
- query.set("distrib", false);
- long results = client.query(COLLECTION, query).getResults().getNumFound();
- if (expected != results) {
- inSync = false;
- replicaNotInSync = solrRunner.getNodeName();
- break;
- }
- }
- }
- if (inSync) return;
- Thread.sleep(500);
- }
-
- fail("Some replicas are not in sync with leader: " + replicaNotInSync);
- }
-
- private void waitForReplicasCatchUp(int numTry) throws IOException, InterruptedException {
- String leaderTimeCommit = getSolrCore(true).get(0).getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
- if (leaderTimeCommit == null) return;
- for (int i = 0; i < numTry; i++) {
- boolean inSync = true;
- for (SolrCore solrCore : getSolrCore(false)) {
- String replicateTimeCommit = solrCore.getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
- if (!leaderTimeCommit.equals(replicateTimeCommit)) {
- inSync = false;
- Thread.sleep(500);
- break;
- }
- }
- if (inSync) return;
- }
-
- fail("Some replicas are not in sync with leader");
-
- }
-
- private List<SolrCore> getSolrCore(boolean isLeader) {
- List<SolrCore> rs = new ArrayList<>();
-
- CloudSolrClient cloudClient = cluster.getSolrClient();
- DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION);
-
- for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
- if (solrRunner.getCoreContainer() == null) continue;
- for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
- CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
- Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
- Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
- if (slice.getLeader() == replica && isLeader) {
- rs.add(solrCore);
- } else if (slice.getLeader() != replica && !isLeader) {
- rs.add(solrCore);
- }
- }
- }
- return rs;
- }
-
- private List<JettySolrRunner> getSolrRunner(boolean isLeader) {
- List<JettySolrRunner> rs = new ArrayList<>();
-
- CloudSolrClient cloudClient = cluster.getSolrClient();
- DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION);
-
- for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
- if (solrRunner.getCoreContainer() == null) continue;
- for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
- CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
- Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
- Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
- if (slice.getLeader() == replica && isLeader) {
- rs.add(solrRunner);
- } else if (slice.getLeader() != replica && !isLeader) {
- rs.add(solrRunner);
- }
- }
- }
- return rs;
- }
-
- // TODO: This is copy/paste from TestPassiveReplica, refactor
- private DocCollection assertNumberOfReplicas(int numWriter, int numActive, int numPassive, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {
- if (updateCollection) {
- cluster.getSolrClient().getZkStateReader().forceUpdateCollection("collection1");
- }
- DocCollection docCollection = getCollectionState("collection1");
- assertNotNull(docCollection);
- assertEquals("Unexpected number of writer replicas: " + docCollection, numWriter,
- docCollection.getReplicas(EnumSet.of(Replica.Type.REALTIME)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
- assertEquals("Unexpected number of passive replicas: " + docCollection, numPassive,
- docCollection.getReplicas(EnumSet.of(Replica.Type.PASSIVE)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
- assertEquals("Unexpected number of active replicas: " + docCollection, numActive,
- docCollection.getReplicas(EnumSet.of(Replica.Type.APPEND)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
- return docCollection;
- }
-
- private CollectionStatePredicate activeReplicaCount(int numWriter, int numActive, int numPassive) {
- return (liveNodes, collectionState) -> {
- int writersFound = 0, activesFound = 0, passivesFound = 0;
- if (collectionState == null)
- return false;
- for (Slice slice : collectionState) {
- for (Replica replica : slice) {
- if (replica.isActive(liveNodes))
- switch (replica.getType()) {
- case APPEND:
- activesFound++;
- break;
- case PASSIVE:
- passivesFound++;
- break;
- case REALTIME:
- writersFound++;
- break;
- default:
- throw new AssertionError("Unexpected replica type");
- }
- }
- }
- return numWriter == writersFound && numActive == activesFound && numPassive == passivesFound;
- };
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1c53272e/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
index b56c5a7..23dde01 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
@@ -24,6 +24,7 @@ import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -50,6 +51,7 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.SolrIndexWriter;
@@ -114,28 +116,6 @@ public class TestAppendReplica extends SolrCloudTestCase {
super.tearDown();
}
- // Just to compare test time, nocommit
- @Ignore
- public void testCreateDelete2() throws Exception {
- try {
- CollectionAdminRequest.createCollection(collectionName, "conf", 1, 8, 0, 0).process(cluster.getSolrClient());
- DocCollection docCollection = getCollectionState(collectionName);
- assertNotNull(docCollection);
-// assertEquals("Expecting 4 relpicas per shard",
-// 8, docCollection.getReplicas().size());
-// assertEquals("Expecting 6 passive replicas, 3 per shard",
-// 6, docCollection.getReplicas(EnumSet.of(Replica.Type.PASSIVE)).size());
-// assertEquals("Expecting 2 writer replicas, one per shard",
-// 2, docCollection.getReplicas(EnumSet.of(Replica.Type.WRITER)).size());
-// for (Slice s:docCollection.getSlices()) {
-// // read-only replicas can never become leaders
-// assertFalse(s.getLeader().isReadOnly());
-// }
- } finally {
- zkClient().printLayoutToStdOut();
- }
- }
-
/**
* Asserts that Update logs exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#REALTIME}, but not
* for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PASSIVE}
@@ -186,6 +166,7 @@ public class TestAppendReplica extends SolrCloudTestCase {
}
}
+ @SuppressWarnings("unchecked")
public void testAddDocs() throws Exception {
int numAppendReplicas = 1 + random().nextInt(3);
DocCollection docCollection = createAndWaitForCollection(1, 0, numAppendReplicas, 0);
@@ -212,9 +193,8 @@ public class TestAppendReplica extends SolrCloudTestCase {
"qt", "/admin/plugins",
"stats", "true");
QueryResponse statsResponse = appendReplicaClient.query(req);
-// TODO: uncomment when SOLR-10569 is fixed
-// assertEquals("Append replicas should recive all updates. Replica: " + r + ", response: " + statsResponse,
-// 1L, ((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats", "cumulative_adds"));
+ assertEquals("Append replicas should recive all updates. Replica: " + r + ", response: " + statsResponse,
+ 1L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.cumulativeAdds.count"));
break;
} catch (AssertionError e) {
if (t.hasTimedOut()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1c53272e/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
index e1f597b..70f2627 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
@@ -63,9 +63,6 @@ public class TestPassiveReplica extends SolrCloudTestCase {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- // TODO: Backup/Snapshot should not work on passive replicas
- // TODO: ADDSHARD operation
-
private String collectionName = null;
private final static int REPLICATION_TIMEOUT_SECS = 10;
@@ -231,10 +228,8 @@ public class TestPassiveReplica extends SolrCloudTestCase {
"qt", "/admin/plugins",
"stats", "true");
QueryResponse statsResponse = readOnlyReplicaClient.query(req);
-// assertEquals("Replicas shouldn't process the add document request: " + statsResponse,
-// 0L, ((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats", "adds"));
assertEquals("Replicas shouldn't process the add document request: " + statsResponse,
- 0L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "/update", "stats")).get("UPDATE./update.requests"));
+ 0L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.adds"));
}
}
assertUlogPresence(docCollection);
[3/6] lucene-solr:jira/solr-10233: Added test coverage for replica
types with Collections API
Posted by tf...@apache.org.
Added test coverage for replica types with Collections API
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d9021e9e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d9021e9e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d9021e9e
Branch: refs/heads/jira/solr-10233
Commit: d9021e9e542e6ec4454720eb3c2b3da881987bc8
Parents: 507548a
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Tue May 9 13:05:50 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Tue May 9 13:05:50 2017 -0700
----------------------------------------------------------------------
.../apache/solr/cloud/TestCollectionAPI.java | 7 +++-
.../apache/solr/cloud/TestPassiveReplica.java | 43 ++++++++++++--------
2 files changed, 33 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d9021e9e/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
index dd55f23..68509b8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
@@ -60,7 +60,12 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
@ShardsFixed(num = 2)
public void test() throws Exception {
try (CloudSolrClient client = createCloudClient(null)) {
- CollectionAdminRequest.Create req = CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1",2,2);
+ CollectionAdminRequest.Create req;
+ if (useAppendReplicas()) {
+ req = CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1",2, 0, 1, 1);
+ } else {
+ req = CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1",2, 1, 0, 1);
+ }
req.setMaxShardsPerNode(2);
client.request(req);
createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d9021e9e/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
index d087c19..e1f597b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
@@ -63,7 +63,6 @@ public class TestPassiveReplica extends SolrCloudTestCase {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- // TODO: Make sure that FORCELEADER can't be used with Passive
// TODO: Backup/Snapshot should not work on passive replicas
// TODO: ADDSHARD operation
@@ -139,22 +138,34 @@ public class TestPassiveReplica extends SolrCloudTestCase {
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1, 0, 3)
.setMaxShardsPerNode(100)
.process(cluster.getSolrClient());
- DocCollection docCollection = getCollectionState(collectionName);
- assertNotNull(docCollection);
- assertEquals("Expecting 4 relpicas per shard",
- 8, docCollection.getReplicas().size());
- assertEquals("Expecting 6 passive replicas, 3 per shard",
- 6, docCollection.getReplicas(EnumSet.of(Replica.Type.PASSIVE)).size());
- assertEquals("Expecting 2 writer replicas, one per shard",
- 2, docCollection.getReplicas(EnumSet.of(Replica.Type.REALTIME)).size());
- for (Slice s:docCollection.getSlices()) {
- // read-only replicas can never become leaders
- assertFalse(s.getLeader().getType() == Replica.Type.PASSIVE);
- List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);
- assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),
- 1, shardElectionNodes.size());
+ boolean reloaded = false;
+ while (true) {
+ DocCollection docCollection = getCollectionState(collectionName);
+ assertNotNull(docCollection);
+ assertEquals("Expecting 4 relpicas per shard",
+ 8, docCollection.getReplicas().size());
+ assertEquals("Expecting 6 passive replicas, 3 per shard",
+ 6, docCollection.getReplicas(EnumSet.of(Replica.Type.PASSIVE)).size());
+ assertEquals("Expecting 2 writer replicas, one per shard",
+ 2, docCollection.getReplicas(EnumSet.of(Replica.Type.REALTIME)).size());
+ for (Slice s:docCollection.getSlices()) {
+ // read-only replicas can never become leaders
+ assertFalse(s.getLeader().getType() == Replica.Type.PASSIVE);
+ List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);
+ assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),
+ 1, shardElectionNodes.size());
+ }
+ assertUlogPresence(docCollection);
+ if (reloaded) {
+ break;
+ } else {
+ // reload
+ CollectionAdminResponse response = CollectionAdminRequest.reloadCollection(collectionName)
+ .process(cluster.getSolrClient());
+ assertEquals(0, response.getStatus());
+ reloaded = true;
+ }
}
- assertUlogPresence(docCollection);
} finally {
zkClient().printLayoutToStdOut();
}
[4/6] lucene-solr:jira/solr-10233: Backup/Restore with replica types
Posted by tf...@apache.org.
Backup/Restore with replica types
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/62712294
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/62712294
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/62712294
Branch: refs/heads/jira/solr-10233
Commit: 6271229493a0f5adac675ee0f0d2c4b2c148cef5
Parents: d9021e9
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed May 10 12:57:40 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed May 10 12:57:40 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/solr/cloud/RestoreCmd.java | 52 ++++++++++++++++++--
.../AbstractCloudBackupRestoreTestCase.java | 23 +++++----
.../apache/solr/common/cloud/DocCollection.java | 24 +++++++++
3 files changed, 85 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62712294/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java
index 4e7fb58..367ee86 100644
--- a/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java
@@ -33,6 +33,7 @@ import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -56,6 +57,7 @@ import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES;
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@@ -178,6 +180,12 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD);
propMap.put(COLLECTION_PROP, restoreCollectionName);
propMap.put(SHARD_ID_PROP, slice.getName());
+
+ if (restoreCollection.getNumRealtimeReplicas() != null && restoreCollection.getNumRealtimeReplicas() >= 1) {
+ propMap.put(REPLICA_TYPE, Replica.Type.REALTIME.name());
+ } else if (restoreCollection.getNumAppendReplicas() != null && restoreCollection.getNumAppendReplicas() >= 1) {
+ propMap.put(REPLICA_TYPE, Replica.Type.APPEND.name());
+ }
// add async param
if (asyncId != null) {
propMap.put(ASYNC, asyncId);
@@ -216,17 +224,51 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
//refresh the location copy of collection state
restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
- //Add the remaining replicas for each shard
- Integer numReplicas = restoreCollection.getReplicationFactor();
- if (numReplicas != null && numReplicas > 1) {
+ //Add the remaining replicas for each shard, considering it's type
+ int numRealtimeReplicas = restoreCollection.getNumRealtimeReplicas() != null?
+ restoreCollection.getNumRealtimeReplicas():0;
+ if (numRealtimeReplicas == 0) {
+ numRealtimeReplicas = restoreCollection.getReplicationFactor() != null?
+ restoreCollection.getReplicationFactor():0;
+ }
+ int numAppendReplicas = restoreCollection.getNumAppendReplicas() != null?
+ restoreCollection.getNumAppendReplicas():0;
+ int numPassiveReplicas = restoreCollection.getNumPassiveReplicas() != null?
+ restoreCollection.getNumPassiveReplicas():0;
+
+ int createdRealtimeReplicas = 0, createdAppendReplicas = 0, createdPassiveReplicas = 0;
+
+ // We already created either a REALTIME or an APPEND replica as leader
+ if (numRealtimeReplicas > 0) {
+ createdRealtimeReplicas++;
+ } else if (createdAppendReplicas > 0) {
+ createdAppendReplicas++;
+ }
+
+ int totalReplicasPerShard = numRealtimeReplicas + numAppendReplicas + numPassiveReplicas;
+
+ if (totalReplicasPerShard > 1) {
log.info("Adding replicas to restored collection={}", restoreCollection);
for (Slice slice : restoreCollection.getSlices()) {
- for (int i = 1; i < numReplicas; i++) {
- log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
+ for (int i = 1; i < totalReplicasPerShard; i++) {
+ Replica.Type typeToCreate;
+ if (createdRealtimeReplicas < numRealtimeReplicas) {
+ createdRealtimeReplicas++;
+ typeToCreate = Replica.Type.REALTIME;
+ } else if (createdAppendReplicas < numAppendReplicas) {
+ createdAppendReplicas++;
+ typeToCreate = Replica.Type.APPEND;
+ } else {
+ createdPassiveReplicas++;
+ typeToCreate = Replica.Type.PASSIVE;
+ }
+
+ log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate);
HashMap<String, Object> propMap = new HashMap<>();
propMap.put(COLLECTION_PROP, restoreCollectionName);
propMap.put(SHARD_ID_PROP, slice.getName());
+ propMap.put(REPLICA_TYPE, typeToCreate.name());
// add async param
if (asyncId != null) {
propMap.put(ASYNC, asyncId);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62712294/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java b/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java
index f39cfed..e9cebc9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AbstractCloudBackupRestoreTestCase.java
@@ -29,6 +29,7 @@ import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.ClusterProp;
import org.apache.solr.client.solrj.response.RequestStatusState;
@@ -45,8 +46,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.params.ShardParams._ROUTE_;
-
/**
* This class implements the logic required to test Solr cloud backup/restore capability.
*/
@@ -81,11 +80,15 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
@Test
public void test() throws Exception {
boolean isImplicit = random().nextBoolean();
+ boolean doSplitShardOperation = !isImplicit && random().nextBoolean();
int replFactor = TestUtil.nextInt(random(), 1, 2);
+ // Split Shard not supported with replica types
+ int numAppendReplicas = TestUtil.nextInt(random(), 0, 1);
+ int numPassiveReplicas = TestUtil.nextInt(random(), 0, 1);
CollectionAdminRequest.Create create =
- CollectionAdminRequest.createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor);
- if (NUM_SHARDS * replFactor > cluster.getJettySolrRunners().size() || random().nextBoolean()) {
- create.setMaxShardsPerNode(NUM_SHARDS);//just to assert it survives the restoration
+ CollectionAdminRequest.createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor, numAppendReplicas, numPassiveReplicas);
+ if (NUM_SHARDS * (replFactor + numAppendReplicas + numPassiveReplicas) > cluster.getJettySolrRunners().size() || random().nextBoolean()) {
+ create.setMaxShardsPerNode((int)Math.ceil(NUM_SHARDS * (replFactor + numAppendReplicas + numPassiveReplicas) / cluster.getJettySolrRunners().size()));//just to assert it survives the restoration
}
if (random().nextBoolean()) {
create.setAutoAddReplicas(true);//just to assert it survives the restoration
@@ -109,7 +112,7 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
indexDocs(getCollectionName());
- if (!isImplicit && random().nextBoolean()) {
+ if (doSplitShardOperation) {
// shard split the first shard
int prevActiveSliceCount = getActiveSliceCount(getCollectionName());
CollectionAdminRequest.SplitShard splitShard = CollectionAdminRequest.splitShard(getCollectionName());
@@ -277,9 +280,11 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
Map<String,Integer> shardToDocCount = new TreeMap<>();
for (Slice slice : docCollection.getActiveSlices()) {
String shardName = slice.getName();
- long docsInShard = client.query(docCollection.getName(), new SolrQuery("*:*").setParam(_ROUTE_, shardName))
- .getResults().getNumFound();
- shardToDocCount.put(shardName, (int) docsInShard);
+ try (HttpSolrClient leaderClient = new HttpSolrClient.Builder(slice.getLeader().getCoreUrl()).withHttpClient(client.getHttpClient()).build()) {
+ long docsInShard = leaderClient.query(new SolrQuery("*:*").setParam("distrib", "false"))
+ .getResults().getNumFound();
+ shardToDocCount.put(shardName, (int) docsInShard);
+ }
}
return shardToDocCount;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/62712294/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
index 3b409b7..3b6ac80 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
@@ -35,6 +35,9 @@ import org.noggit.JSONWriter;
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+import static org.apache.solr.common.cloud.ZkStateReader.REALTIME_REPLICAS;
+import static org.apache.solr.common.cloud.ZkStateReader.APPEND_REPLICAS;
+import static org.apache.solr.common.cloud.ZkStateReader.PASSIVE_REPLICAS;
/**
* Models a Collection in zookeeper (but that Java name is obviously taken, hence "DocCollection")
@@ -58,6 +61,9 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice> {
private final String znode;
private final Integer replicationFactor;
+ private final Integer numRealtimeReplicas;
+ private final Integer numAppendReplicas;
+ private final Integer numPassiveReplicas;
private final Integer maxShardsPerNode;
private final Boolean autoAddReplicas;
@@ -81,6 +87,9 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice> {
this.nodeNameLeaderReplicas = new HashMap<>();
this.nodeNameReplicas = new HashMap<>();
this.replicationFactor = (Integer) verifyProp(props, REPLICATION_FACTOR);
+ this.numRealtimeReplicas = (Integer) verifyProp(props, REALTIME_REPLICAS);
+ this.numAppendReplicas = (Integer) verifyProp(props, APPEND_REPLICAS);
+ this.numPassiveReplicas = (Integer) verifyProp(props, PASSIVE_REPLICAS);
this.maxShardsPerNode = (Integer) verifyProp(props, MAX_SHARDS_PER_NODE);
Boolean autoAddReplicas = (Boolean) verifyProp(props, AUTO_ADD_REPLICAS);
this.autoAddReplicas = autoAddReplicas == null ? Boolean.FALSE : autoAddReplicas;
@@ -127,6 +136,9 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice> {
switch (propName) {
case MAX_SHARDS_PER_NODE:
case REPLICATION_FACTOR:
+ case REALTIME_REPLICAS:
+ case PASSIVE_REPLICAS:
+ case APPEND_REPLICAS:
return Integer.parseInt(o.toString());
case AUTO_ADD_REPLICAS:
return Boolean.parseBoolean(o.toString());
@@ -330,4 +342,16 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice> {
return super.equals(that) && Objects.equals(this.znode, other.znode) && this.znodeVersion == other.znodeVersion;
}
+ public Integer getNumRealtimeReplicas() {
+ return numRealtimeReplicas;
+ }
+
+ public Integer getNumAppendReplicas() {
+ return numAppendReplicas;
+ }
+
+ public Integer getNumPassiveReplicas() {
+ return numPassiveReplicas;
+ }
+
}