You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/09/28 02:19:01 UTC
[24/29] lucene-solr:jira/http2: SOLR-12756: Refactor Assign and
extract replica placement strategies out of it.
SOLR-12756: Refactor Assign and extract replica placement strategies out of it.
Now, assignment is done with the help of a builder class instead of calling a method with large number of arguments. The number of special cases that had to be handled have been cut down as well.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c587410f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c587410f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c587410f
Branch: refs/heads/jira/http2
Commit: c587410f99375005c680ece5e24a4dfd40d8d3eb
Parents: a6d39ba
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Thu Sep 27 16:15:38 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Sep 27 16:15:38 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +
.../cloud/api/collections/AddReplicaCmd.java | 45 +--
.../solr/cloud/api/collections/Assign.java | 358 ++++++++++++-------
.../api/collections/CreateCollectionCmd.java | 34 +-
.../cloud/api/collections/ReplaceNodeCmd.java | 33 +-
.../solr/cloud/api/collections/RestoreCmd.java | 16 +-
.../cloud/api/collections/SplitShardCmd.java | 17 +-
.../solr/cloud/overseer/ReplicaMutator.java | 4 +-
.../CollectionTooManyReplicasTest.java | 8 +-
.../solr/cloud/autoscaling/TestPolicyCloud.java | 1 +
.../sim/SimClusterStateProvider.java | 24 +-
.../autoscaling/sim/TestSimPolicyCloud.java | 1 +
.../solr/common/cloud/ReplicaPosition.java | 2 +-
13 files changed, 331 insertions(+), 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 98fb204..914fa7c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -105,6 +105,10 @@ Other Changes
* SOLR-12762: Fix javadoc for SolrCloudTestCase.clusterShape() method and add a method that validates only against
Active slices (Anshum Gupta)
+* SOLR-12756: Refactor Assign and extract replica placement strategies out of it. Now, assignment is done with the help
+ of a builder class instead of calling a method with large number of arguments. The number of special cases that had
+ to be handled have been cut down as well. (shalin)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
index f128c2e..6e851db 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
@@ -34,10 +34,8 @@ import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.cloud.ActiveReplicaWatcher;
-import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.common.SolrCloseableLatch;
import org.apache.solr.common.SolrException;
@@ -76,6 +74,12 @@ import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STA
public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ /**
+ * When AddReplica is called with this set to true, then we do not try to find node assignments
+ * for the add replica API. If set to true, a valid "node" should be specified.
+ */
+ public static final String SKIP_NODE_ASSIGNMENT = "skipNodeAssignment";
+
private final OverseerCollectionMessageHandler ocmh;
public AddReplicaCmd(OverseerCollectionMessageHandler ocmh) {
@@ -213,6 +217,8 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
ZkStateReader.COLLECTION_PROP, withCollectionName,
ZkStateReader.SHARD_ID_PROP, withCollectionShard,
"node", createReplica.node,
+ // since we already computed node assignments (which include assigning a node for this withCollection replica) we want to skip the assignment step
+ SKIP_NODE_ASSIGNMENT, "true",
CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.TRUE.toString()); // set to true because we want `withCollection` to be ready after this collection is created
addReplica(clusterState, props, results, null);
}
@@ -300,7 +306,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
}
- log.info("Node Identified {} for creating new replica of shard {}", node, shard);
+ log.info("Node Identified {} for creating new replica of shard {} for collection {}", node, shard, collection);
if (!clusterState.liveNodesContain(node)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
}
@@ -327,6 +333,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
EnumMap<Replica.Type, Integer> replicaTypeVsCount,
AtomicReference< PolicyHelper.SessionWrapper> sessionWrapper) throws IOException, InterruptedException {
boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);
+ boolean skipNodeAssignment = message.getBool(SKIP_NODE_ASSIGNMENT, false);
String sliceName = message.getStr(SHARD_ID_PROP);
DocCollection collection = clusterState.getCollection(collectionName);
@@ -345,33 +352,11 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
}
List<ReplicaPosition> positions = null;
- if (!skipCreateReplicaInClusterState) {
- if (CloudUtil.usePolicyFramework(collection, cloudManager)) {
- if (node == null) {
- if (collection.getPolicyName() != null) message.getProperties().put(Policy.POLICY, collection.getPolicyName());
- positions = Assign.identifyNodes(cloudManager,
- clusterState,
- Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM),
- collection.getName(),
- message,
- Collections.singletonList(sliceName),
- numNrtReplicas,
- numTlogReplicas,
- numPullReplicas);
- sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
- }
- } else {
- List<Assign.ReplicaCount> sortedNodeList = Assign.getNodesForNewReplicas(clusterState, collection.getName(), sliceName, numNrtReplicas,
- numTlogReplicas, numPullReplicas, createNodeSetStr, cloudManager);
- int i = 0;
- positions = new ArrayList<>();
- for (Map.Entry<Replica.Type, Integer> e : replicaTypeVsCount.entrySet()) {
- for (int j = 0; j < e.getValue(); j++) {
- positions.add(new ReplicaPosition(sliceName, j + 1, e.getKey(), sortedNodeList.get(i % sortedNodeList.size()).nodeName));
- i++;
- }
- }
- }
+ if (!skipCreateReplicaInClusterState && !skipNodeAssignment) {
+
+ positions = Assign.getNodesForNewReplicas(clusterState, collection.getName(), sliceName, numNrtReplicas,
+ numTlogReplicas, numPullReplicas, createNodeSetStr, cloudManager);
+ sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
}
if (positions == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
index 9b33f52..542ca1b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
@@ -21,12 +21,14 @@ import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;
@@ -217,12 +219,7 @@ public class Assign {
}
public static List<String> getLiveOrLiveAndCreateNodeSetList(final Set<String> liveNodes, final ZkNodeProps message, final Random random) {
- // TODO: add smarter options that look at the current number of cores per
- // node?
- // for now we just go random (except when createNodeSet and createNodeSet.shuffle=false are passed in)
-
List<String> nodeList;
-
final String createNodeSetStr = message.getStr(CREATE_NODE_SET);
final List<String> createNodeList = (createNodeSetStr == null) ? null :
StrUtils.splitSmart((OverseerCollectionMessageHandler.CREATE_NODE_SET_EMPTY.equals(createNodeSetStr) ?
@@ -243,67 +240,6 @@ public class Assign {
return nodeList;
}
- public static List<ReplicaPosition> identifyNodes(SolrCloudManager cloudManager,
- ClusterState clusterState,
- List<String> nodeList,
- String collectionName,
- ZkNodeProps message,
- List<String> shardNames,
- int numNrtReplicas,
- int numTlogReplicas,
- int numPullReplicas) throws IOException, InterruptedException, AssignmentException {
- List<Map> rulesMap = (List) message.get("rule");
- String policyName = message.getStr(POLICY);
- AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
-
- if (rulesMap == null && policyName == null && autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) {
- log.debug("Identify nodes using default");
- int i = 0;
- List<ReplicaPosition> result = new ArrayList<>();
- for (String aShard : shardNames)
- for (Map.Entry<Replica.Type, Integer> e : ImmutableMap.of(Replica.Type.NRT, numNrtReplicas,
- Replica.Type.TLOG, numTlogReplicas,
- Replica.Type.PULL, numPullReplicas
- ).entrySet()) {
- for (int j = 0; j < e.getValue(); j++){
- result.add(new ReplicaPosition(aShard, j, e.getKey(), nodeList.get(i % nodeList.size())));
- i++;
- }
- }
- return result;
- } else {
- if (numTlogReplicas + numPullReplicas != 0 && rulesMap != null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- Replica.Type.TLOG + " or " + Replica.Type.PULL + " replica types not supported with placement rules");
- }
- }
-
- if (rulesMap != null && !rulesMap.isEmpty()) {
- List<Rule> rules = new ArrayList<>();
- for (Object map : rulesMap) rules.add(new Rule((Map) map));
- Map<String, Integer> sharVsReplicaCount = new HashMap<>();
-
- for (String shard : shardNames) sharVsReplicaCount.put(shard, numNrtReplicas);
- ReplicaAssigner replicaAssigner = new ReplicaAssigner(rules,
- sharVsReplicaCount,
- (List<Map>) message.get(SNITCH),
- new HashMap<>(),//this is a new collection. So, there are no nodes in any shard
- nodeList,
- cloudManager,
- clusterState);
-
- Map<ReplicaPosition, String> nodeMappings = replicaAssigner.getNodeMappings();
- return nodeMappings.entrySet().stream()
- .map(e -> new ReplicaPosition(e.getKey().shard, e.getKey().index, e.getKey().type, e.getValue()))
- .collect(Collectors.toList());
- } else {
- if (message.getStr(CREATE_NODE_SET) == null)
- nodeList = Collections.emptyList();// unless explicitly specified do not pass node list to Policy
- return getPositionsUsingPolicy(collectionName,
- shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas, policyName, cloudManager, nodeList);
- }
- }
-
static class ReplicaCount {
public final String nodeName;
public int thisCollectionNodes = 0;
@@ -318,11 +254,11 @@ public class Assign {
}
}
- // Only called from createShard and addReplica (so far).
+ // Only called from addReplica (and by extension createShard) (so far).
//
// Gets a list of candidate nodes to put the required replica(s) on. Throws errors if not enough replicas
// could be created on live nodes given maxShardsPerNode, Replication factor (if from createShard) etc.
- public static List<ReplicaCount> getNodesForNewReplicas(ClusterState clusterState, String collectionName,
+ public static List<ReplicaPosition> getNodesForNewReplicas(ClusterState clusterState, String collectionName,
String shard, int nrtReplicas, int tlogReplicas, int pullReplicas,
Object createNodeSet, SolrCloudManager cloudManager) throws IOException, InterruptedException, AssignmentException {
log.debug("getNodesForNewReplicas() shard: {} , nrtReplicas : {} , tlogReplicas: {} , pullReplicas: {} , createNodeSet {}", shard, nrtReplicas, tlogReplicas, pullReplicas, createNodeSet );
@@ -331,13 +267,13 @@ public class Assign {
List<String> createNodeList = null;
if (createNodeSet instanceof List) {
- createNodeList = (List) createNodeSet;
+ createNodeList = (List<String>) createNodeSet;
} else {
// deduplicate
createNodeList = createNodeSet == null ? null : new ArrayList<>(new LinkedHashSet<>(StrUtils.splitSmart((String) createNodeSet, ",", true)));
}
- HashMap<String, ReplicaCount> nodeNameVsShardCount = getNodeNameVsShardCount(collectionName, clusterState, createNodeList);
+ HashMap<String, ReplicaCount> nodeNameVsShardCount = getNodeNameVsShardCount(collectionName, clusterState, createNodeList);
if (createNodeList == null) { // We only care if we haven't been told to put new replicas on specific nodes.
long availableSlots = 0;
@@ -349,40 +285,22 @@ public class Assign {
}
if (availableSlots < nrtReplicas + tlogReplicas + pullReplicas) {
throw new AssignmentException(
- String.format(Locale.ROOT, "Cannot create %d new replicas for collection %s given the current number of live nodes and a maxShardsPerNode of %d",
- nrtReplicas, collectionName, maxShardsPerNode));
+ String.format(Locale.ROOT, "Cannot create %d new replicas for collection %s given the current number of eligible live nodes %d and a maxShardsPerNode of %d",
+ nrtReplicas, collectionName, nodeNameVsShardCount.size(), maxShardsPerNode));
}
}
- List l = (List) coll.get(DocCollection.RULE);
- List<ReplicaPosition> replicaPositions = null;
- if (l != null) {
- if (tlogReplicas + pullReplicas > 0) {
- throw new AssignmentException(Replica.Type.TLOG + " or " + Replica.Type.PULL +
- " replica types not supported with placement rules");
- }
- // TODO: make it so that this method doesn't require access to CC
- replicaPositions = getNodesViaRules(clusterState, shard, nrtReplicas, cloudManager, coll, createNodeList, l);
- }
- String policyName = coll.getStr(POLICY);
- AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
- if (policyName != null || !autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) {
- replicaPositions = Assign.getPositionsUsingPolicy(collectionName, Collections.singletonList(shard), nrtReplicas, tlogReplicas, pullReplicas,
- policyName, cloudManager, createNodeList);
- }
-
- if(replicaPositions != null){
- List<ReplicaCount> repCounts = new ArrayList<>();
- for (ReplicaPosition p : replicaPositions) {
- repCounts.add(new ReplicaCount(p.node));
- }
- return repCounts;
- }
-
- ArrayList<ReplicaCount> sortedNodeList = new ArrayList<>(nodeNameVsShardCount.values());
- Collections.sort(sortedNodeList, (x, y) -> (x.weight() < y.weight()) ? -1 : ((x.weight() == y.weight()) ? 0 : 1));
- return sortedNodeList;
-
+ AssignRequest assignRequest = new AssignRequestBuilder()
+ .forCollection(collectionName)
+ .forShard(Collections.singletonList(shard))
+ .assignNrtReplicas(nrtReplicas)
+ .assignTlogReplicas(tlogReplicas)
+ .assignPullReplicas(pullReplicas)
+ .onNodes(createNodeList)
+ .build();
+ AssignStrategyFactory assignStrategyFactory = new AssignStrategyFactory(cloudManager);
+ AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, coll);
+ return assignStrategy.assign(cloudManager, assignRequest);
}
public static List<ReplicaPosition> getPositionsUsingPolicy(String collName, List<String> shardNames,
@@ -418,35 +336,7 @@ public class Assign {
}
}
- private static List<ReplicaPosition> getNodesViaRules(ClusterState clusterState, String shard, int numberOfNodes,
- SolrCloudManager cloudManager, DocCollection coll, List<String> createNodeList, List l) {
- ArrayList<Rule> rules = new ArrayList<>();
- for (Object o : l) rules.add(new Rule((Map) o));
- Map<String, Map<String, Integer>> shardVsNodes = new LinkedHashMap<>();
- for (Slice slice : coll.getSlices()) {
- LinkedHashMap<String, Integer> n = new LinkedHashMap<>();
- shardVsNodes.put(slice.getName(), n);
- for (Replica replica : slice.getReplicas()) {
- Integer count = n.get(replica.getNodeName());
- if (count == null) count = 0;
- n.put(replica.getNodeName(), ++count);
- }
- }
- List snitches = (List) coll.get(SNITCH);
- List<String> nodesList = createNodeList == null ?
- new ArrayList<>(clusterState.getLiveNodes()) :
- createNodeList;
- Map<ReplicaPosition, String> positions = new ReplicaAssigner(
- rules,
- Collections.singletonMap(shard, numberOfNodes),
- snitches,
- shardVsNodes,
- nodesList, cloudManager, clusterState).getNodeMappings();
-
- return positions.entrySet().stream().map(e -> e.getKey().setNode(e.getValue())).collect(Collectors.toList());// getReplicaCounts(positions);
- }
-
- private static HashMap<String, ReplicaCount> getNodeNameVsShardCount(String collectionName,
+ static HashMap<String, ReplicaCount> getNodeNameVsShardCount(String collectionName,
ClusterState clusterState, List<String> createNodeList) {
Set<String> nodes = clusterState.getLiveNodes();
@@ -477,7 +367,7 @@ public class Assign {
for (Replica replica : replicas) {
ReplicaCount count = nodeNameVsShardCount.get(replica.getNodeName());
if (count != null) {
- count.totalNodes++; // Used ot "weigh" whether this node should be used later.
+ count.totalNodes++; // Used to "weigh" whether this node should be used later.
if (entry.getKey().equals(collectionName)) {
count.thisCollectionNodes++;
if (count.thisCollectionNodes >= maxShardsPerNode) nodeNameVsShardCount.remove(replica.getNodeName());
@@ -513,4 +403,210 @@ public class Assign {
super(message, cause, enableSuppression, writableStackTrace);
}
}
+
+ public interface AssignStrategy {
+ List<ReplicaPosition> assign(SolrCloudManager solrCloudManager, AssignRequest assignRequest)
+ throws Assign.AssignmentException, IOException, InterruptedException;
+ }
+
+ public static class AssignRequest {
+ public String collectionName;
+ public List<String> shardNames;
+ public List<String> nodes;
+ public int numNrtReplicas;
+ public int numTlogReplicas;
+ public int numPullReplicas;
+
+ public AssignRequest(String collectionName, List<String> shardNames, List<String> nodes, int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
+ this.collectionName = collectionName;
+ this.shardNames = shardNames;
+ this.nodes = nodes;
+ this.numNrtReplicas = numNrtReplicas;
+ this.numTlogReplicas = numTlogReplicas;
+ this.numPullReplicas = numPullReplicas;
+ }
+ }
+
+ public static class AssignRequestBuilder {
+ private String collectionName;
+ private List<String> shardNames;
+ private List<String> nodes;
+ private int numNrtReplicas;
+ private int numTlogReplicas;
+ private int numPullReplicas;
+
+ public AssignRequestBuilder forCollection(String collectionName) {
+ this.collectionName = collectionName;
+ return this;
+ }
+
+ public AssignRequestBuilder forShard(List<String> shardNames) {
+ this.shardNames = shardNames;
+ return this;
+ }
+
+ public AssignRequestBuilder onNodes(List<String> nodes) {
+ this.nodes = nodes;
+ return this;
+ }
+
+ public AssignRequestBuilder assignNrtReplicas(int numNrtReplicas) {
+ this.numNrtReplicas = numNrtReplicas;
+ return this;
+ }
+
+ public AssignRequestBuilder assignTlogReplicas(int numTlogReplicas) {
+ this.numTlogReplicas = numTlogReplicas;
+ return this;
+ }
+
+ public AssignRequestBuilder assignPullReplicas(int numPullReplicas) {
+ this.numPullReplicas = numPullReplicas;
+ return this;
+ }
+
+ public AssignRequest build() {
+ Objects.requireNonNull(collectionName, "The collectionName cannot be null");
+ Objects.requireNonNull(shardNames, "The shard names cannot be null");
+ return new AssignRequest(collectionName, shardNames, nodes, numNrtReplicas,
+ numTlogReplicas, numPullReplicas);
+ }
+ }
+
+ public static class LegacyAssignStrategy implements AssignStrategy {
+ @Override
+ public List<ReplicaPosition> assign(SolrCloudManager solrCloudManager, AssignRequest assignRequest) throws Assign.AssignmentException, IOException, InterruptedException {
+ ClusterState clusterState = solrCloudManager.getClusterStateProvider().getClusterState();
+ List<String> nodeList = assignRequest.nodes;
+
+ HashMap<String, Assign.ReplicaCount> nodeNameVsShardCount = Assign.getNodeNameVsShardCount(assignRequest.collectionName, clusterState, assignRequest.nodes);
+ if (nodeList == null || nodeList.isEmpty()) {
+ ArrayList<Assign.ReplicaCount> sortedNodeList = new ArrayList<>(nodeNameVsShardCount.values());
+ sortedNodeList.sort(Comparator.comparingInt(Assign.ReplicaCount::weight));
+ nodeList = sortedNodeList.stream().map(replicaCount -> replicaCount.nodeName).collect(Collectors.toList());
+ }
+
+ int i = 0;
+ List<ReplicaPosition> result = new ArrayList<>();
+ for (String aShard : assignRequest.shardNames)
+ for (Map.Entry<Replica.Type, Integer> e : ImmutableMap.of(Replica.Type.NRT, assignRequest.numNrtReplicas,
+ Replica.Type.TLOG, assignRequest.numTlogReplicas,
+ Replica.Type.PULL, assignRequest.numPullReplicas
+ ).entrySet()) {
+ for (int j = 0; j < e.getValue(); j++) {
+ result.add(new ReplicaPosition(aShard, j, e.getKey(), nodeList.get(i % nodeList.size())));
+ i++;
+ }
+ }
+ return result;
+ }
+ }
+
+ public static class RulesBasedAssignStrategy implements AssignStrategy {
+ public List<Rule> rules;
+ public List snitches;
+ public ClusterState clusterState;
+
+ public RulesBasedAssignStrategy(List<Rule> rules, List snitches, ClusterState clusterState) {
+ this.rules = rules;
+ this.snitches = snitches;
+ this.clusterState = clusterState;
+ }
+
+ @Override
+ public List<ReplicaPosition> assign(SolrCloudManager solrCloudManager, AssignRequest assignRequest) throws Assign.AssignmentException, IOException, InterruptedException {
+ if (assignRequest.numTlogReplicas + assignRequest.numPullReplicas != 0) {
+ throw new Assign.AssignmentException(
+ Replica.Type.TLOG + " or " + Replica.Type.PULL + " replica types not supported with placement rules or cluster policies");
+ }
+
+ Map<String, Integer> shardVsReplicaCount = new HashMap<>();
+ for (String shard : assignRequest.shardNames) shardVsReplicaCount.put(shard, assignRequest.numNrtReplicas);
+
+ Map<String, Map<String, Integer>> shardVsNodes = new LinkedHashMap<>();
+ DocCollection docCollection = solrCloudManager.getClusterStateProvider().getClusterState().getCollectionOrNull(assignRequest.collectionName);
+ if (docCollection != null) {
+ for (Slice slice : docCollection.getSlices()) {
+ LinkedHashMap<String, Integer> n = new LinkedHashMap<>();
+ shardVsNodes.put(slice.getName(), n);
+ for (Replica replica : slice.getReplicas()) {
+ Integer count = n.get(replica.getNodeName());
+ if (count == null) count = 0;
+ n.put(replica.getNodeName(), ++count);
+ }
+ }
+ }
+
+ List<String> nodesList = assignRequest.nodes == null ? new ArrayList<>(clusterState.getLiveNodes()) : assignRequest.nodes;
+
+ ReplicaAssigner replicaAssigner = new ReplicaAssigner(rules,
+ shardVsReplicaCount,
+ snitches,
+ shardVsNodes,
+ nodesList,
+ solrCloudManager, clusterState);
+
+ Map<ReplicaPosition, String> nodeMappings = replicaAssigner.getNodeMappings();
+ return nodeMappings.entrySet().stream()
+ .map(e -> new ReplicaPosition(e.getKey().shard, e.getKey().index, e.getKey().type, e.getValue()))
+ .collect(Collectors.toList());
+ }
+ }
+
+ public static class PolicyBasedAssignStrategy implements AssignStrategy {
+ public String policyName;
+
+ public PolicyBasedAssignStrategy(String policyName) {
+ this.policyName = policyName;
+ }
+
+ @Override
+ public List<ReplicaPosition> assign(SolrCloudManager solrCloudManager, AssignRequest assignRequest) throws Assign.AssignmentException, IOException, InterruptedException {
+ return Assign.getPositionsUsingPolicy(assignRequest.collectionName,
+ assignRequest.shardNames, assignRequest.numNrtReplicas,
+ assignRequest.numTlogReplicas, assignRequest.numPullReplicas,
+ policyName, solrCloudManager, assignRequest.nodes);
+ }
+ }
+
+ public static class AssignStrategyFactory {
+ public SolrCloudManager solrCloudManager;
+
+ public AssignStrategyFactory(SolrCloudManager solrCloudManager) {
+ this.solrCloudManager = solrCloudManager;
+ }
+
+ public AssignStrategy create(ClusterState clusterState, DocCollection collection) throws IOException, InterruptedException {
+ List<Map> ruleMaps = (List<Map>) collection.get("rule");
+ String policyName = collection.getStr(POLICY);
+ List snitches = (List) collection.get(SNITCH);
+ AutoScalingConfig autoScalingConfig = solrCloudManager.getDistribStateManager().getAutoScalingConfig();
+
+ StrategyType strategyType = null;
+ if ((ruleMaps == null || ruleMaps.isEmpty()) && policyName == null && autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) {
+ strategyType = StrategyType.LEGACY;
+ } else if (ruleMaps != null && !ruleMaps.isEmpty()) {
+ strategyType = StrategyType.RULES;
+ } else {
+ strategyType = StrategyType.POLICY;
+ }
+
+ switch (strategyType) {
+ case LEGACY:
+ return new LegacyAssignStrategy();
+ case RULES:
+ List<Rule> rules = new ArrayList<>();
+ for (Object map : ruleMaps) rules.add(new Rule((Map) map));
+ return new RulesBasedAssignStrategy(rules, snitches, clusterState);
+ case POLICY:
+ return new PolicyBasedAssignStrategy(policyName);
+ default:
+ throw new Assign.AssignmentException("Unknown strategy type: " + strategyType);
+ }
+ }
+
+ private enum StrategyType {
+ LEGACY, RULES, POLICY;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 4f66ff9..542345d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -42,6 +42,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
+import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
@@ -131,7 +132,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
ocmh.validateConfigOrThrowSolrException(configName);
- List<String> nodeList = new ArrayList<>();
String router = message.getStr("router.name", DocRouter.DEFAULT_NAME);
String policy = message.getStr(Policy.POLICY);
AutoScalingConfig autoScalingConfig = ocmh.cloudManager.getDistribStateManager().getAutoScalingConfig();
@@ -177,10 +177,12 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not fully create collection: " + collectionName);
}
+ // refresh cluster state
+ clusterState = ocmh.cloudManager.getClusterStateProvider().getClusterState();
+
List<ReplicaPosition> replicaPositions = null;
try {
- replicaPositions = buildReplicaPositions(ocmh.cloudManager, clusterState, message,
- nodeList, shardNames, sessionWrapper);
+ replicaPositions = buildReplicaPositions(ocmh.cloudManager, clusterState, clusterState.getCollection(collectionName), message, shardNames, sessionWrapper);
} catch (Assign.AssignmentException e) {
ZkNodeProps deleteMessage = new ZkNodeProps("name", collectionName);
new DeleteCollectionCmd(ocmh).call(clusterState, deleteMessage, results);
@@ -188,7 +190,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
throw new SolrException(ErrorCode.SERVER_ERROR, e.getMessage(), e.getCause());
}
- if (nodeList.isEmpty()) {
+ if (replicaPositions.isEmpty()) {
log.debug("Finished create command for collection: {}", collectionName);
return;
}
@@ -333,8 +335,9 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
public static List<ReplicaPosition> buildReplicaPositions(SolrCloudManager cloudManager, ClusterState clusterState,
+ DocCollection docCollection,
ZkNodeProps message,
- List<String> nodeList, List<String> shardNames,
+ List<String> shardNames,
AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper) throws IOException, InterruptedException, Assign.AssignmentException {
final String collectionName = message.getStr(NAME);
// look at the replication factor and see if it matches reality
@@ -342,11 +345,9 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
int numTlogReplicas = message.getInt(TLOG_REPLICAS, 0);
int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, numTlogReplicas>0?0:1));
int numPullReplicas = message.getInt(PULL_REPLICAS, 0);
- AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
- String policy = message.getStr(Policy.POLICY);
- boolean usePolicyFramework = !autoScalingConfig.getPolicy().getClusterPolicy().isEmpty() || policy != null;
+ boolean usePolicyFramework = CloudUtil.usePolicyFramework(docCollection, cloudManager);
- Integer numSlices = shardNames.size();
+ int numSlices = shardNames.size();
int maxShardsPerNode = checkMaxShardsPerNode(message, usePolicyFramework);
// we need to look at every node and see how many cores it serves
@@ -354,7 +355,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
// but (for now) require that each core goes on a distinct node.
List<ReplicaPosition> replicaPositions;
- nodeList.addAll(Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM));
+ List<String> nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM);
if (nodeList.isEmpty()) {
log.warn("It is unusual to create a collection ("+collectionName+") without cores.");
@@ -387,8 +388,17 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
+ ". This requires " + requestedShardsToCreate
+ " shards to be created (higher than the allowed number)");
}
- replicaPositions = Assign.identifyNodes(cloudManager
- , clusterState, nodeList, collectionName, message, shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas);
+ Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
+ .forCollection(collectionName)
+ .forShard(shardNames)
+ .assignNrtReplicas(numNrtReplicas)
+ .assignTlogReplicas(numTlogReplicas)
+ .assignPullReplicas(numPullReplicas)
+ .onNodes(nodeList)
+ .build();
+ Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(cloudManager);
+ Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, docCollection);
+ replicaPositions = assignStrategy.assign(cloudManager, assignRequest);
sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true));
}
return replicaPositions;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
index a09eec3..c622f0f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
@@ -104,20 +104,25 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
try {
for (ZkNodeProps sourceReplica : sourceReplicas) {
NamedList nl = new NamedList();
- log.info("Going to create replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target);
+ String sourceCollection = sourceReplica.getStr(COLLECTION_PROP);
+ log.info("Going to create replica for collection={} shard={} on node={}", sourceCollection, sourceReplica.getStr(SHARD_ID_PROP), target);
String targetNode = target;
if (targetNode == null) {
Replica.Type replicaType = Replica.Type.get(sourceReplica.getStr(ZkStateReader.REPLICA_TYPE));
- targetNode = Assign.identifyNodes(ocmh.cloudManager,
- clusterState,
- new ArrayList<>(ocmh.cloudManager.getClusterStateProvider().getLiveNodes()),
- sourceReplica.getStr(COLLECTION_PROP),
- message,
- Collections.singletonList(sourceReplica.getStr(SHARD_ID_PROP)),
- replicaType == Replica.Type.NRT ? 1: 0,
- replicaType == Replica.Type.TLOG ? 1 : 0,
- replicaType == Replica.Type.PULL ? 1 : 0
- ).get(0).node;
+ int numNrtReplicas = replicaType == Replica.Type.NRT ? 1 : 0;
+ int numTlogReplicas = replicaType == Replica.Type.TLOG ? 1 : 0;
+ int numPullReplicas = replicaType == Replica.Type.PULL ? 1 : 0;
+ Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
+ .forCollection(sourceCollection)
+ .forShard(Collections.singletonList(sourceReplica.getStr(SHARD_ID_PROP)))
+ .assignNrtReplicas(numNrtReplicas)
+ .assignTlogReplicas(numTlogReplicas)
+ .assignPullReplicas(numPullReplicas)
+ .onNodes(new ArrayList<>(ocmh.cloudManager.getClusterStateProvider().getLiveNodes()))
+ .build();
+ Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager);
+ Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, clusterState.getCollection(sourceCollection));
+ targetNode = assignStrategy.assign(ocmh.cloudManager, assignRequest).get(0).node;
sessionWrapperRef.set(PolicyHelper.getLastSessionWrapper(true));
}
ZkNodeProps msg = sourceReplica.plus("parallel", String.valueOf(parallel)).plus(CoreAdminParams.NODE, targetNode);
@@ -127,7 +132,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
countDownLatch.countDown();
if (nl.get("failure") != null) {
String errorString = String.format(Locale.ROOT, "Failed to create replica for collection=%s shard=%s" +
- " on node=%s", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target);
+ " on node=%s", sourceCollection, sourceReplica.getStr(SHARD_ID_PROP), target);
log.warn(errorString);
// one replica creation failed. Make the best attempt to
// delete all the replicas created so far in the target
@@ -138,7 +143,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
}
} else {
log.debug("Successfully created replica for collection={} shard={} on node={}",
- sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target);
+ sourceCollection, sourceReplica.getStr(SHARD_ID_PROP), target);
}
}).get(0);
@@ -147,7 +152,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
if (sourceReplica.getBool(ZkStateReader.LEADER_PROP, false) || waitForFinalState) {
String shardName = sourceReplica.getStr(SHARD_ID_PROP);
String replicaName = sourceReplica.getStr(ZkStateReader.REPLICA_PROP);
- String collectionName = sourceReplica.getStr(COLLECTION_PROP);
+ String collectionName = sourceCollection;
String key = collectionName + "_" + replicaName;
CollectionStateWatcher watcher;
if (waitForFinalState) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
index d082ac3..d100ce0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
@@ -232,11 +232,17 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
PolicyHelper.SessionWrapper sessionWrapper = null;
try {
- List<ReplicaPosition> replicaPositions = Assign.identifyNodes(
- ocmh.cloudManager, clusterState,
- nodeList, restoreCollectionName,
- message, sliceNames,
- numNrtReplicas, numTlogReplicas, numPullReplicas);
+ Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
+ .forCollection(restoreCollectionName)
+ .forShard(sliceNames)
+ .assignNrtReplicas(numNrtReplicas)
+ .assignTlogReplicas(numTlogReplicas)
+ .assignPullReplicas(numPullReplicas)
+ .onNodes(nodeList)
+ .build();
+ Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager);
+ Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, restoreCollection);
+ List<ReplicaPosition> replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest);
sessionWrapper = PolicyHelper.getLastSessionWrapper(true);
//Create one replica per shard and copy backed up data to it
for (Slice slice : restoreCollection.getSlices()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 00488a3..bac45ab 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -361,12 +361,17 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
}
t = timings.sub("identifyNodesForReplicas");
- List<ReplicaPosition> replicaPositions = Assign.identifyNodes(ocmh.cloudManager,
- clusterState,
- new ArrayList<>(clusterState.getLiveNodes()),
- collectionName,
- new ZkNodeProps(collection.getProperties()),
- subSlices, numNrt.get(), numTlog.get(), numPull.get());
+ Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
+ .forCollection(collectionName)
+ .forShard(subSlices)
+ .assignNrtReplicas(numNrt.get())
+ .assignTlogReplicas(numTlog.get())
+ .assignPullReplicas(numPull.get())
+ .onNodes(new ArrayList<>(clusterState.getLiveNodes()))
+ .build();
+ Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager);
+ Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, collection);
+ List<ReplicaPosition> replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest);
sessionWrapper = PolicyHelper.getLastSessionWrapper(true);
t.stop();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
index 34843c1..6cbdbfb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
@@ -282,9 +282,7 @@ public class ReplicaMutator {
Slice slice = collection != null ? collection.getSlice(sliceName) : null;
- Map<String, Object> replicaProps = new LinkedHashMap<>();
-
- replicaProps.putAll(message.getProperties());
+ Map<String, Object> replicaProps = new LinkedHashMap<>(message.getProperties());
if (slice != null) {
Replica oldReplica = slice.getReplica(coreNodeName);
if (oldReplica != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java
index 09a119b..1ac0cae 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java
@@ -79,7 +79,7 @@ public class CollectionTooManyReplicasTest extends SolrCloudTestCase {
});
assertTrue("Should have gotten the right error message back",
- e.getMessage().contains("given the current number of live nodes and a maxShardsPerNode of"));
+ e.getMessage().contains("given the current number of eligible live nodes"));
// Oddly, we should succeed next just because setting property.name will not check for nodes being "full up"
@@ -106,7 +106,7 @@ public class CollectionTooManyReplicasTest extends SolrCloudTestCase {
});
assertTrue("Should have gotten the right error message back",
- e2.getMessage().contains("given the current number of live nodes and a maxShardsPerNode of"));
+ e2.getMessage().contains("given the current number of eligible live nodes"));
// wait for recoveries to finish, for a clean shutdown - see SOLR-9645
waitForState("Expected to see all replicas active", collectionName, (n, c) -> {
@@ -141,7 +141,7 @@ public class CollectionTooManyReplicasTest extends SolrCloudTestCase {
.process(cluster.getSolrClient());
});
assertTrue("Should have gotten the right error message back",
- e.getMessage().contains("given the current number of live nodes and a maxShardsPerNode of"));
+ e.getMessage().contains("given the current number of eligible live nodes"));
// Hmmm, providing a nodeset also overrides the checks for max replicas, so prove it.
List<String> nodes = getAllNodeNames(collectionName);
@@ -156,7 +156,7 @@ public class CollectionTooManyReplicasTest extends SolrCloudTestCase {
.process(cluster.getSolrClient());
});
assertTrue("Should have gotten the right error message back",
- e2.getMessage().contains("given the current number of live nodes and a maxShardsPerNode of"));
+ e2.getMessage().contains("given the current number of eligible live nodes"));
// And finally, ensure that there are all the replicas we expect. We should have shards 1, 2 and 4 and each
// should have exactly two replicas
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
index d1dcecf..bfd5878 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
@@ -325,6 +325,7 @@ public class TestPolicyCloud extends SolrCloudTestCase {
Utils.getObjectByPath(json, true, "cluster-policy[2]/port"));
CollectionAdminRequest.createCollectionWithImplicitRouter("policiesTest", "conf", "s1", 1, 1, 1)
+ .setMaxShardsPerNode(-1)
.process(cluster.getSolrClient());
DocCollection coll = getCollectionState("policiesTest");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 4b73200..08ce6bf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -741,7 +741,6 @@ public class SimClusterStateProvider implements ClusterStateProvider {
results.add(CoreAdminParams.REQUESTID, props.getStr(CommonAdminParams.ASYNC));
}
boolean waitForFinalState = props.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false);
- List<String> nodeList = new ArrayList<>();
final String collectionName = props.getStr(NAME);
String router = props.getStr("router.name", DocRouter.DEFAULT_NAME);
@@ -784,8 +783,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
opDelay(collectionName, CollectionParams.CollectionAction.CREATE.name());
AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper = new AtomicReference<>();
- List<ReplicaPosition> replicaPositions = CreateCollectionCmd.buildReplicaPositions(cloudManager, getClusterState(), props,
- nodeList, shardNames, sessionWrapper);
+ List<ReplicaPosition> replicaPositions = CreateCollectionCmd.buildReplicaPositions(cloudManager, getClusterState(), cmd.collection, props,
+ shardNames, sessionWrapper);
if (sessionWrapper.get() != null) {
sessionWrapper.get().release();
}
@@ -1102,13 +1101,18 @@ public class SimClusterStateProvider implements ClusterStateProvider {
SplitShardCmd.fillRanges(cloudManager, message, collection, parentSlice, subRanges, subSlices, subShardNames, true);
// add replicas for new subShards
int repFactor = parentSlice.getReplicas().size();
- List<ReplicaPosition> replicaPositions = Assign.identifyNodes(cloudManager,
- clusterState,
- new ArrayList<>(clusterState.getLiveNodes()),
- collectionName,
- new ZkNodeProps(collection.getProperties()),
- // reproduce the bug
- subSlices, repFactor, 0, 0);
+ Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
+ .forCollection(collectionName)
+ .forShard(subSlices)
+ .assignNrtReplicas(repFactor)
+ .assignTlogReplicas(0)
+ .assignPullReplicas(0)
+ .onNodes(new ArrayList<>(clusterState.getLiveNodes()))
+ .build();
+ Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(cloudManager);
+ Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, collection);
+ // reproduce the bug
+ List<ReplicaPosition> replicaPositions = assignStrategy.assign(cloudManager, assignRequest);
PolicyHelper.SessionWrapper sessionWrapper = PolicyHelper.getLastSessionWrapper(true);
if (sessionWrapper != null) sessionWrapper.release();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
index 3637428..ffdfff7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
@@ -257,6 +257,7 @@ public class TestSimPolicyCloud extends SimSolrCloudTestCase {
Utils.getObjectByPath(json, true, "cluster-policy[2]/port"));
CollectionAdminRequest.createCollectionWithImplicitRouter("policiesTest", "conf", "s1", 1, 1, 1)
+ .setMaxShardsPerNode(-1)
.process(solrClient);
CloudTestUtils.waitForState(cluster, "Timeout waiting for collection to become active", "policiesTest",
CloudTestUtils.clusterShape(1, 3, false, true));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c587410f/solr/solrj/src/java/org/apache/solr/common/cloud/ReplicaPosition.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ReplicaPosition.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ReplicaPosition.java
index 591a001..62d8761 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ReplicaPosition.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ReplicaPosition.java
@@ -40,7 +40,7 @@ public class ReplicaPosition implements Comparable<ReplicaPosition> {
public int compareTo(ReplicaPosition that) {
//this is to ensure that we try one replica from each shard first instead of
// all replicas from same shard
- return that.index > index ? -1 : that.index == index ? 0 : 1;
+ return Integer.compare(index, that.index);
}
@Override