You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2018/10/09 07:02:23 UTC
lucene-solr:branch_7x: SOLR-12739: Autoscaling policy framework is
now used as the default strategy to select the nodes on which new replicas or
replicas of new collections are created.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x a1f03ba43 -> ee09bef75
SOLR-12739: Autoscaling policy framework is now used as the default strategy to select the nodes on which new replicas or replicas of new collections are created.
Previously, the maxShardsPerNode parameter was not allowed on collections when autoscaling policy was configured. Also if an autoscaling policy was configured then the default was to set an unlimited maxShardsPerNode automatically. Now the maxShardsPerNode parameter is always allowed during collection creation and maxShardsPerNode should be set correctly (if required) regardless of whether autoscaling policies are in effect or not. The default value of maxShardsPerNode continues to be 1 as before. It can be set to -1 during collection creation to fall back to the old behavior of unlimited maxShardsPerNode when using autoscaling policy. This patch also fixes PolicyHelper to find the free disk space requirements of a new replica from the leader only if said leader node is alive.
(cherry picked from commit dbed8bafe6ee167361599deaa4f1b5fdbb0b1c32)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ee09bef7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ee09bef7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ee09bef7
Branch: refs/heads/branch_7x
Commit: ee09bef75d12f88c3e5616d769816e79a81518f1
Parents: a1f03ba
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Oct 9 12:10:28 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Oct 9 12:32:13 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 11 +++
.../java/org/apache/solr/cloud/CloudUtil.java | 37 -----------
.../solr/cloud/api/collections/Assign.java | 70 +++++++++++++++++---
.../api/collections/CreateCollectionCmd.java | 19 +-----
.../api/collections/DeleteCollectionCmd.java | 2 +
.../solr/cloud/CollectionsAPISolrJTest.java | 11 +++
...verseerCollectionConfigSetProcessorTest.java | 7 ++
.../solr/cloud/api/collections/AssignTest.java | 39 ++++++++++-
.../autoscaling/AutoScalingHandlerTest.java | 13 +---
.../autoscaling/ComputePlanActionTest.java | 10 +--
.../autoscaling/ExecutePlanActionTest.java | 1 +
.../sim/SimClusterStateProvider.java | 4 +-
.../sim/TestSimComputePlanAction.java | 4 +-
solr/solr-ref-guide/src/collections-api.adoc | 4 +-
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
.../client/solrj/cloud/autoscaling/Policy.java | 2 +
.../solrj/cloud/autoscaling/PolicyHelper.java | 2 +-
.../common/params/CollectionAdminParams.java | 10 ++-
.../src/resources/apispec/cluster.Commands.json | 9 +++
.../solrj/cloud/autoscaling/TestPolicy.java | 24 +------
20 files changed, 171 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a755161..90bd4ab 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -38,6 +38,15 @@ Upgrade Notes
ZK as well as written using the V2 set-obj-property syntax but it is deprecated and will be removed in Solr 9.
We recommend that users change their API calls to use the new format going forward.
+* SOLR-12739: Autoscaling policy framework is now used as the default strategy to select the nodes on which
+ new replicas or replicas of new collections are created. Previously, the maxShardsPerNode parameter was not allowed
+ on collections when autoscaling policy was configured. Also if an autoscaling policy was configured then the default
+ was to set an unlimited maxShardsPerNode automatically. Now the maxShardsPerNode parameter is always
+ allowed during collection creation and maxShardsPerNode should be set correctly (if required) regardless of whether
+ autoscaling policies are in effect or not. The default value of maxShardsPerNode continues to be 1 as before. It can
+ be set to -1 during collection creation to fall back to the old behavior of unlimited maxShardsPerNode when using
+ autoscaling policy.
+
New Features
----------------------
@@ -108,6 +117,8 @@ Improvements
can't be uninverted (saves mem) and can avoid wrapping the reader altogether if there's nothing to uninvert.
IndexSchema.getUninversionMap refactored to getUninversionMapper and no longer merges FieldInfos. (David Smiley)
+* SOLR-12739: Make autoscaling policy based replica placement the default strategy for placing replicas. (shalin)
+
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
index 14bebef..302703b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
@@ -23,12 +23,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
import org.apache.commons.io.FileUtils;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
@@ -146,37 +142,4 @@ public class CloudUtil {
}
- /**
- * <b>Note:</b> where possible, the {@link #usePolicyFramework(DocCollection, SolrCloudManager)} method should
- * be used instead of this method
- *
- * @return true if autoscaling policy framework should be used for replica placement
- */
- public static boolean usePolicyFramework(SolrCloudManager cloudManager) throws IOException, InterruptedException {
- Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null");
- return usePolicyFramework(Optional.empty(), cloudManager);
- }
-
- /**
- * @return true if auto scaling policy framework should be used for replica placement
- * for this collection, otherwise false
- */
- public static boolean usePolicyFramework(DocCollection collection, SolrCloudManager cloudManager)
- throws IOException, InterruptedException {
- Objects.requireNonNull(collection, "The DocCollection instance cannot be null");
- Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null");
- return usePolicyFramework(Optional.of(collection), cloudManager);
- }
-
- private static boolean usePolicyFramework(Optional<DocCollection> collection, SolrCloudManager cloudManager) throws IOException, InterruptedException {
- AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
- // if no autoscaling configuration exists then obviously we cannot use the policy framework
- if (autoScalingConfig.getPolicy().isEmpty()) return false;
- // do custom preferences exist
- if (!autoScalingConfig.getPolicy().isEmptyPreferences()) return true;
- // does a cluster policy exist
- if (!autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) return true;
- // finally we check if the current collection has a policy
- return !collection.isPresent() || collection.get().getPolicyName() != null;
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
index a24ad1a..841ee93 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;
@@ -41,7 +42,6 @@ import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
-import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
@@ -52,6 +52,7 @@ import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.NumberUtils;
@@ -241,6 +242,56 @@ public class Assign {
return nodeList;
}
+ /**
+ * <b>Note:</b> where possible, the {@link #usePolicyFramework(DocCollection, SolrCloudManager)} method should
+ * be used instead of this method
+ *
+ * @return true if autoscaling policy framework should be used for replica placement
+ */
+ public static boolean usePolicyFramework(SolrCloudManager cloudManager) throws IOException, InterruptedException {
+ Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null");
+ return usePolicyFramework(Optional.empty(), cloudManager);
+ }
+
+ /**
+ * @return true if auto scaling policy framework should be used for replica placement
+ * for this collection, otherwise false
+ */
+ public static boolean usePolicyFramework(DocCollection collection, SolrCloudManager cloudManager)
+ throws IOException, InterruptedException {
+ Objects.requireNonNull(collection, "The DocCollection instance cannot be null");
+ Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null");
+ return usePolicyFramework(Optional.of(collection), cloudManager);
+ }
+
+ private static boolean usePolicyFramework(Optional<DocCollection> collection, SolrCloudManager cloudManager) throws IOException, InterruptedException {
+ boolean useLegacyAssignment = false;
+ Map<String, Object> clusterProperties = cloudManager.getClusterStateProvider().getClusterProperties();
+ if (clusterProperties.containsKey(CollectionAdminParams.DEFAULTS)) {
+ Map<String, Object> defaults = (Map<String, Object>) clusterProperties.get(CollectionAdminParams.DEFAULTS);
+ Map<String, Object> collectionDefaults = (Map<String, Object>) defaults.getOrDefault(CollectionAdminParams.COLLECTION, Collections.emptyMap());
+ useLegacyAssignment = (boolean) collectionDefaults.getOrDefault(CollectionAdminParams.USE_LEGACY_REPLICA_ASSIGNMENT, false);
+ }
+
+ if (!useLegacyAssignment) {
+ // if legacy assignment is not selected then autoscaling is always available through the implicit policy/preferences
+ return true;
+ }
+
+ // legacy assignment is turned on, which means we must look at the actual autoscaling config
+ // to determine whether policy framework can be used or not for this collection
+
+ AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
+ // if no autoscaling configuration exists then obviously we cannot use the policy framework
+ if (autoScalingConfig.getPolicy().isEmpty()) return false;
+ // do custom preferences exist
+ if (!autoScalingConfig.getPolicy().isEmptyPreferences()) return true;
+ // does a cluster policy exist
+ if (!autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) return true;
+ // finally we check if the current collection has a policy
+ return !collection.isPresent() || collection.get().getPolicyName() != null;
+ }
+
static class ReplicaCount {
public final String nodeName;
public int thisCollectionNodes = 0;
@@ -581,18 +632,17 @@ public class Assign {
List<Map> ruleMaps = (List<Map>) collection.get("rule");
String policyName = collection.getStr(POLICY);
List snitches = (List) collection.get(SNITCH);
- AutoScalingConfig autoScalingConfig = solrCloudManager.getDistribStateManager().getAutoScalingConfig();
- StrategyType strategyType = null;
- if ((ruleMaps == null || ruleMaps.isEmpty()) && !CloudUtil.usePolicyFramework(collection, solrCloudManager)) {
- strategyType = StrategyType.LEGACY;
+ Strategy strategy = null;
+ if ((ruleMaps == null || ruleMaps.isEmpty()) && !usePolicyFramework(collection, solrCloudManager)) {
+ strategy = Strategy.LEGACY;
} else if (ruleMaps != null && !ruleMaps.isEmpty()) {
- strategyType = StrategyType.RULES;
+ strategy = Strategy.RULES;
} else {
- strategyType = StrategyType.POLICY;
+ strategy = Strategy.POLICY;
}
- switch (strategyType) {
+ switch (strategy) {
case LEGACY:
return new LegacyAssignStrategy();
case RULES:
@@ -602,11 +652,11 @@ public class Assign {
case POLICY:
return new PolicyBasedAssignStrategy(policyName);
default:
- throw new Assign.AssignmentException("Unknown strategy type: " + strategyType);
+ throw new Assign.AssignmentException("Unknown strategy type: " + strategy);
}
}
- private enum StrategyType {
+ private enum Strategy {
LEGACY, RULES, POLICY;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 212437e..533aee8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -38,10 +38,8 @@ import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException;
import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException;
import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
-import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
@@ -132,12 +130,9 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
ocmh.validateConfigOrThrowSolrException(configName);
String router = message.getStr("router.name", DocRouter.DEFAULT_NAME);
- String policy = message.getStr(Policy.POLICY);
- boolean usePolicyFramework = CloudUtil.usePolicyFramework(ocmh.cloudManager) || policy != null;
// fail fast if parameters are wrong or incomplete
List<String> shardNames = populateShardNames(message, router);
- checkMaxShardsPerNode(message, usePolicyFramework);
checkReplicaTypes(message);
AtomicReference<PolicyHelper.SessionWrapper> sessionWrapper = new AtomicReference<>();
@@ -343,10 +338,10 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
int numTlogReplicas = message.getInt(TLOG_REPLICAS, 0);
int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, numTlogReplicas>0?0:1));
int numPullReplicas = message.getInt(PULL_REPLICAS, 0);
- boolean usePolicyFramework = CloudUtil.usePolicyFramework(docCollection, cloudManager);
int numSlices = shardNames.size();
- int maxShardsPerNode = checkMaxShardsPerNode(message, usePolicyFramework);
+ int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
+ if (maxShardsPerNode == -1) maxShardsPerNode = Integer.MAX_VALUE;
// we need to look at every node and see how many cores it serves
// add our new cores to existing nodes serving the least number of cores
@@ -402,16 +397,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
return replicaPositions;
}
- public static int checkMaxShardsPerNode(ZkNodeProps message, boolean usePolicyFramework) {
- int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
- if (usePolicyFramework && message.getStr(MAX_SHARDS_PER_NODE) != null && maxShardsPerNode > 0) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "'maxShardsPerNode>0' is not supported when autoScaling policies are used");
- }
- if (maxShardsPerNode == -1 || usePolicyFramework) maxShardsPerNode = Integer.MAX_VALUE;
-
- return maxShardsPerNode;
- }
-
public static void checkReplicaTypes(ZkNodeProps message) {
int numTlogReplicas = message.getInt(TLOG_REPLICAS, 0);
int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, numTlogReplicas > 0 ? 0 : 1));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
index 9a569d1..f1767ee 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -180,6 +181,7 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
}
private String referencedByAlias(String collection, Aliases aliases) {
+ Objects.requireNonNull(aliases);
return aliases.getCollectionAliasListMap().entrySet().stream()
.filter(e -> e.getValue().contains(collection))
.map(Map.Entry::getKey) // alias name
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 406c5e6..b9c7e3a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -43,12 +43,15 @@ import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -56,6 +59,7 @@ import static java.util.Arrays.asList;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.DEFAULTS;
@@ -69,6 +73,13 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
.configure();
}
+ @Before
+ public void beforeTest() throws Exception {
+ // clear any persisted auto scaling configuration
+ zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
+ cluster.deleteAllCollections();
+ }
+
/**
* When a config name is not specified during collection creation, the _default should
* be used.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
index 241cfb2..e999d73 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
@@ -43,6 +43,7 @@ import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.cloud.Overseer.LeaderStatus;
import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler;
+import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
@@ -76,6 +77,9 @@ import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
+import static org.apache.solr.common.params.CollectionAdminParams.DEFAULTS;
+import static org.apache.solr.common.params.CollectionAdminParams.USE_LEGACY_REPLICA_ASSIGNMENT;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyBoolean;
import static org.mockito.Mockito.anyInt;
@@ -260,6 +264,7 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
when(zkStateReaderMock.getZkClient()).thenReturn(solrZkClientMock);
when(zkStateReaderMock.getClusterState()).thenReturn(clusterStateMock);
when(zkStateReaderMock.getAutoScalingConfig()).thenReturn(autoScalingConfig);
+ when(zkStateReaderMock.getAliases()).thenReturn(Aliases.EMPTY);
when(clusterStateMock.getCollection(anyString())).thenAnswer(invocation -> {
String key = invocation.getArgument(0);
@@ -325,6 +330,8 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
when(zkControllerMock.getSolrCloudManager()).thenReturn(cloudDataProviderMock);
when(cloudDataProviderMock.getClusterStateProvider()).thenReturn(clusterStateProviderMock);
when(clusterStateProviderMock.getClusterState()).thenReturn(clusterStateMock);
+ when(clusterStateProviderMock.getLiveNodes()).thenReturn(liveNodes);
+ when(clusterStateProviderMock.getClusterProperties()).thenReturn(Utils.makeMap(DEFAULTS, Utils.makeMap(COLLECTION, Utils.makeMap(USE_LEGACY_REPLICA_ASSIGNMENT, true))));
when(cloudDataProviderMock.getDistribStateManager()).thenReturn(stateManagerMock);
when(stateManagerMock.hasData(anyString())).thenAnswer(invocation -> zkMap.containsKey(invocation.getArgument(0)));
when(stateManagerMock.getAutoScalingConfig()).thenReturn(autoScalingConfig);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java
index d2b35e4..91e37b2 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java
@@ -18,14 +18,21 @@ package org.apache.solr.cloud.api.collections;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.stream.Collectors;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.cloud.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.impl.ZkDistribStateManager;
import org.apache.solr.cloud.ZkTestServer;
import org.apache.solr.common.cloud.DocCollection;
@@ -34,6 +41,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.Before;
@@ -152,5 +160,34 @@ public class AssignTest extends SolrTestCaseJ4 {
server.shutdown();
}
}
-
+
+ @Test
+ public void testUsePolicyByDefault() throws Exception {
+ assumeWorkingMockito();
+
+ SolrCloudManager solrCloudManager = mock(SolrCloudManager.class);
+ ClusterStateProvider clusterStateProvider = mock(ClusterStateProvider.class);
+ when(solrCloudManager.getClusterStateProvider()).thenReturn(clusterStateProvider);
+ // first we set useLegacyReplicaAssignment=false, so autoscaling should always be used
+ when(clusterStateProvider.getClusterProperties()).thenReturn(Utils.makeMap("defaults", Utils.makeMap("collection", Utils.makeMap("useLegacyReplicaAssignment", false))));
+ // verify
+ boolean usePolicyFramework = Assign.usePolicyFramework(solrCloudManager);
+ assertTrue(usePolicyFramework);
+
+ // now we set useLegacyReplicaAssignment=true, so autoscaling can only be used if an explicit policy or preference exists
+ when(clusterStateProvider.getClusterProperties()).thenReturn(Utils.makeMap("defaults", Utils.makeMap("collection", Utils.makeMap("useLegacyReplicaAssignment", true))));
+ DistribStateManager distribStateManager = mock(DistribStateManager.class);
+ when(solrCloudManager.getDistribStateManager()).thenReturn(distribStateManager);
+ when(distribStateManager.getAutoScalingConfig()).thenReturn(new AutoScalingConfig(Collections.emptyMap()));
+ assertFalse(Assign.usePolicyFramework(solrCloudManager));
+
+ // lets provide a custom preference and assert that autoscaling is used even if useLegacyReplicaAssignment=false
+ // our custom preferences are exactly the same as the default ones
+ // but because we are providing them explicitly, they must cause autoscaling to turn on
+ List<Map> customPreferences = Policy.DEFAULT_PREFERENCES
+ .stream().map(preference -> preference.getOriginal()).collect(Collectors.toList());
+
+ when(distribStateManager.getAutoScalingConfig()).thenReturn(new AutoScalingConfig(Utils.makeMap("cluster-preferences", customPreferences)));
+ assertTrue(Assign.usePolicyFramework(solrCloudManager));
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 5c469c9..d8219ab 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -777,16 +777,6 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertNotNull(violations);
assertEquals(0, violations.size());
- // assert that when a cluster policy is in effect, using maxShardsPerNode throws an exception
- try {
- CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6);
- create.setMaxShardsPerNode(10);
- create.process(solrClient);
- fail();
- } catch (Exception e) {
- assertTrue(e.getMessage().contains("'maxShardsPerNode>0' is not supported when autoScaling policies are used"));
- }
-
// temporarily increase replica limit in cluster policy so that we can create a collection with 6 replicas
String tempClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
@@ -799,7 +789,8 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertEquals(response.get("result").toString(), "success");
// lets create a collection which violates the rule replicas < 2
- CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6);
+ CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6)
+ .setMaxShardsPerNode(3);
CollectionAdminResponse adminResponse = create.process(solrClient);
assertTrue(adminResponse.isSuccess());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 2eaec83..087094a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -276,7 +276,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeWithMultipleReplicasLost",
"conf",2, 3);
-// create.setMaxShardsPerNode(2);
+ create.setMaxShardsPerNode(2);
create.process(solrClient);
waitForState("Timed out waiting for replicas of new collection to be active",
@@ -353,7 +353,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
assertEquals(response.get("result").toString(), "success");
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeAdded",
- "conf",1, 2);
+ "conf",1, 2).setMaxShardsPerNode(2);
create.process(solrClient);
waitForState("Timed out waiting for replicas of new collection to be active",
@@ -553,7 +553,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0",
- "conf", numShards, 1);
+ "conf", numShards, 1).setMaxShardsPerNode(2);
create.process(solrClient);
waitForState("Timed out waiting for replicas of new collection to be active",
@@ -579,7 +579,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
for (int i = 1; i < numCollections; i++) {
create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_" + i,
- "conf", numShards, 2);
+ "conf", numShards, 2).setMaxShardsPerNode(numShards * 2);
create.process(solrClient);
waitForState("Timed out waiting for replicas of new collection to be active",
@@ -649,7 +649,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
String newNodeName = newNode.getNodeName();
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0",
- "conf", numShards, 2);
+ "conf", numShards, 2).setMaxShardsPerNode(numShards * 2);
create.process(solrClient);
waitForState("Timed out waiting for replicas of new collection to be active",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
index c876557..c15bc53 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
@@ -88,6 +88,7 @@ public class ExecutePlanActionTest extends SolrCloudTestCase {
cluster.waitForAllNodes(30);
loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
+ cluster.deleteAllCollections();
}
@Test
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index ee39666..1d377b6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -99,6 +99,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
@@ -787,7 +788,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
// fail fast if parameters are wrong or incomplete
List<String> shardNames = CreateCollectionCmd.populateShardNames(props, router);
- CreateCollectionCmd.checkMaxShardsPerNode(props, usePolicyFramework);
+ int maxShardsPerNode = props.getInt(MAX_SHARDS_PER_NODE, 1);
+ if (maxShardsPerNode == -1) maxShardsPerNode = Integer.MAX_VALUE;
CreateCollectionCmd.checkReplicaTypes(props);
// always force getting fresh state
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java
index 719bb7b..98a7728 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java
@@ -205,7 +205,7 @@ public class TestSimComputePlanAction extends SimSolrCloudTestCase {
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeWithMultipleReplicasLost",
"conf",2, 3);
-// create.setMaxShardsPerNode(2);
+ create.setMaxShardsPerNode(2);
create.process(solrClient);
CloudTestUtils.waitForState(cluster, "Timed out waiting for replicas of new collection to be active",
@@ -283,7 +283,7 @@ public class TestSimComputePlanAction extends SimSolrCloudTestCase {
assertEquals(response.get("result").toString(), "success");
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeAdded",
- "conf",1, 4);
+ "conf",1, 4).setMaxShardsPerNode(-1);
create.process(solrClient);
CloudTestUtils.waitForState(cluster, "Timed out waiting for replicas of new collection to be active",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index f320cdb..0e4112d 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -70,7 +70,7 @@ When creating collections, the shards and/or replicas are spread across all avai
+
If a node is not live when the CREATE action is called, it will not get any parts of the new collection, which could lead to too many replicas being created on a single live node. Defining `maxShardsPerNode` sets a limit on the number of replicas the CREATE action will spread to each node.
+
-If the entire collection can not be fit into the live nodes, no collection will be created at all. The default `maxShardsPerNode` value is `1`.
+If the entire collection can not be fit into the live nodes, no collection will be created at all. The default `maxShardsPerNode` value is `1`. A value of `-1` means unlimited. If a `policy` is also specified then the stricter of `maxShardsPerNode` and policy rules apply.
`createNodeSet`::
Allows defining the nodes to spread the new collection across. The format is a comma-separated list of node_names, such as `localhost:8983_solr,localhost:8984_solr,localhost:8985_solr`.
@@ -80,7 +80,7 @@ If not provided, the CREATE operation will create shard-replicas spread across a
Alternatively, use the special value of `EMPTY` to initially create no shard-replica within the new collection and then later use the <<addreplica,ADDREPLICA>> operation to add shard-replicas when and where required.
`createNodeSet.shuffle`::
-Controls wether or not the shard-replicas created for this collection will be assigned to the nodes specified by the `createNodeSet` in a sequential manner, or if the list of nodes should be shuffled prior to creating individual replicas.
+Controls whether or not the shard-replicas created for this collection will be assigned to the nodes specified by the `createNodeSet` in a sequential manner, or if the list of nodes should be shuffled prior to creating individual replicas.
+
A `false` value makes the results of a collection creation predictable and gives more exact control over the location of the individual shard-replicas, but `true` can be a better choice for ensuring replicas are distributed evenly across nodes. The default is `true`.
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
index 518d9ee..279a396 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
@@ -53,7 +53,7 @@ Cluster preferences allow you to tell Solr how to assess system load on each nod
In general, when an operation increases replica counts, the *least loaded* <<solrcloud-autoscaling-policy-preferences.adoc#node-selector,qualified node>> will be chosen, and when the operation reduces replica counts, the *most loaded* qualified node will be chosen.
-The default cluster preferences are `[{minimize:cores}]`, which tells Solr to minimize the number of cores on all nodes. In this case, the least loaded node is the one with the fewest cores.
+The default cluster preferences are `[{minimize:cores},{maximize:freedisk}]`, which tells Solr to minimize the number of cores on all nodes and if number of cores are equal, maximize the free disk space available. In this case, the least loaded node is the one with the fewest cores or if two nodes have an equal number of cores, the node with the most free disk space.
You can learn more about preferences in the <<solrcloud-autoscaling-policy-preferences.adoc#solrcloud-autoscaling-policy-preferences,Autoscaling Cluster Preferences>> section.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index cfe9455..db1015f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -80,6 +80,8 @@ public class Policy implements MapWriter {
public static final Set<String> GLOBAL_ONLY_TAGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("cores", CollectionAdminParams.WITH_COLLECTION)));
public static final List<Preference> DEFAULT_PREFERENCES = Collections.unmodifiableList(
Arrays.asList(
+ // NOTE - if you change this, make sure to update the solrcloud-autoscaling-overview.adoc which
+ // lists the default preferences
new Preference((Map<String, Object>) Utils.fromJSONString("{minimize : cores, precision:1}")),
new Preference((Map<String, Object>) Utils.fromJSONString("{maximize : freedisk}"))));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 5e88df1..d7a7a22 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -130,7 +130,7 @@ public class PolicyHelper {
if (coll != null) {
for (String shardName : shardNames) {
Replica ldr = coll.getLeader(shardName);
- if (ldr != null) {
+ if (ldr != null && cloudManager.getClusterStateProvider().getLiveNodes().contains(ldr.getNodeName())) {
Map<String, Map<String, List<ReplicaInfo>>> details = cloudManager.getNodeStateProvider().getReplicaInfo(ldr.getNodeName(),
Collections.singleton(FREEDISK.perReplicaValue));
ReplicaInfo replicaInfo = details.getOrDefault(collName, emptyMap()).getOrDefault(shardName, singletonList(null)).get(0);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
index cb70fb8..c34f930 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
@@ -92,7 +92,15 @@ public interface CollectionAdminParams {
String COLOCATED_WITH = "COLOCATED_WITH";
/**
- * Used by cluster properties API to provide defaults for collection, cluster etc.
+ * Used by cluster properties API as a wrapper key to provide defaults for collection, cluster etc.
+ *
+ * e.g. {defaults:{collection:{useLegacyReplicaAssignment:false}}}
*/
String DEFAULTS = "defaults";
+
+ /**
+ * This cluster property decides whether Solr should use the legacy round-robin replica placement strategy
+ * or the autoscaling policy based strategy to assign replicas to nodes. The default is false.
+ */
+ String USE_LEGACY_REPLICA_ASSIGNMENT = "useLegacyReplicaAssignment";
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/solrj/src/resources/apispec/cluster.Commands.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json
index d1f5738..da31b3c 100644
--- a/solr/solrj/src/resources/apispec/cluster.Commands.json
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.json
@@ -93,6 +93,15 @@
"defaults" : {
"type" : "object",
"properties": {
+ "cluster": {
+ "type" : "object",
+ "properties": {
+ "useLegacyReplicaAssignment": {
+ "type" : "boolean",
+ "description" : "Decides wheyher to use the deprecated legacy replica assignment strategy or not"
+ }
+ }
+ },
"collection": {
"type": "object",
"properties": {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee09bef7/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index ec3c56c..fecc749 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -49,7 +49,6 @@ import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.api.collections.Assign;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.ClusterState;
@@ -57,7 +56,6 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
-import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.params.CollectionParams;
@@ -81,9 +79,6 @@ import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.CLUSTER_PREF
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.REPLICA;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -3091,13 +3086,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
" 'totaldisk': 1700," +
" 'port': 8985" +
" }" +
- " }," +
- " 'autoscalingJson': {" +
- " 'cluster-preferences': [" +
- " { 'maximize': 'freedisk'}," +
- " { 'minimize': 'cores', 'precision': 3}" +
- " ]" +
- " }" +
+ " }" +
"}";
String clusterState = "{\n" +
@@ -3161,7 +3150,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
});
});
- AutoScalingConfig asc = m.containsKey("autoscalingJson") ? new AutoScalingConfig((Map<String, Object>) m.get("autoscalingJson")) : null;
+ AutoScalingConfig asc = m.containsKey("autoscalingJson") ? new AutoScalingConfig((Map<String, Object>) m.get("autoscalingJson")) : new AutoScalingConfig(Collections.emptyMap());
DelegatingCloudManager cloudManager = new DelegatingCloudManager(null) {
@Override
@@ -3207,13 +3196,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
};
- ZkNodeProps message = new ZkNodeProps(
- Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(),
- COLLECTION_PROP, "c1",
- SHARD_ID_PROP, "s1",
- REPLICA_TYPE, Replica.Type.NRT.toString()
- );
-
Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
.forCollection("c1")
.forShard(Collections.singletonList("s1"))
@@ -3237,7 +3219,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
* The reason behind doing this is to ensure that implicitly added cluster preferences do not ever
* go to ZooKeeper so that we can decide whether to enable autoscaling policy framework or not.
*
- * @see org.apache.solr.cloud.CloudUtil#usePolicyFramework(DocCollection, SolrCloudManager)
+ * @see Assign#usePolicyFramework(DocCollection, SolrCloudManager)
*/
public void testPolicyMapWriterWithEmptyPreferences() throws IOException {
List<Map> defaultPreferences = Policy.DEFAULT_PREFERENCES