You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/09/28 02:18:50 UTC
[13/29] lucene-solr:jira/http2: SOLR-11522: /autoscaling/suggestions
now include rebalance options as well even if there are no violations
SOLR-11522: /autoscaling/suggestions now include rebalance options as well even if there are no violations
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3f2975c2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3f2975c2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3f2975c2
Branch: refs/heads/jira/http2
Commit: 3f2975c27c520ed7543779cd731a7cd525243479
Parents: baf40d5
Author: noble <no...@apache.org>
Authored: Tue Sep 25 00:34:21 2018 +1000
Committer: noble <no...@apache.org>
Committed: Tue Sep 25 00:34:21 2018 +1000
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../cloud/autoscaling/FreeDiskVariable.java | 12 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 87 +++++++++----
.../client/solrj/cloud/autoscaling/Row.java | 2 +-
.../solrj/cloud/autoscaling/Suggester.java | 2 +-
.../solrj/cloud/autoscaling/Suggestion.java | 5 +
.../solr/common/ConditionalMapWriter.java | 2 +-
.../java/org/apache/solr/common/MapWriter.java | 5 +
.../java/org/apache/solr/common/util/Utils.java | 31 ++++-
.../autoscaling/testSuggestionsRebalance2.json | 130 +++++++++++++++++++
.../testSuggestionsRebalanceOnly.json | 105 +++++++++++++++
.../solrj/cloud/autoscaling/TestPolicy.java | 98 +++++++-------
.../solrj/cloud/autoscaling/TestPolicy2.java | 39 ++++++
13 files changed, 431 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 2c4903b..6128986 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -95,6 +95,8 @@ New Features
As part of this change, the CREATESHARD API now delegates placing replicas entirely to the ADDREPLICA command
and uses the new parameters to add all the replicas in one API call. (shalin)
+* SOLR-11522: /autoscaling/suggestions now include rebalance options as well even if there are no violations (noble)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
index 600695a..778a837 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
@@ -18,9 +18,9 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Comparator;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -104,19 +104,17 @@ public class FreeDiskVariable extends VariableBase {
}
}
} else if (ctx.violation.replicaCountDelta < 0) {
- suggestNegativeViolations(ctx, shards -> getSortedShards(ctx,shards));
+ suggestNegativeViolations(ctx, shards -> getSortedShards(ctx.session.matrix, shards, ctx.violation.coll));
}
}
-
-
- private List<String> getSortedShards(Suggestion.Ctx ctx, Set<String> shardSet) {
+ static List<String> getSortedShards(List<Row> matrix, Collection<String> shardSet, String coll) {
return shardSet.stream()
.map(shard1 -> {
AtomicReference<Pair<String, Long>> result = new AtomicReference<>();
- for (Row node : ctx.session.matrix) {
- node.forEachShard(ctx.violation.coll, (s, ri) -> {
+ for (Row node : matrix) {
+ node.forEachShard(coll, (s, ri) -> {
if (result.get() != null) return;
if (s.equals(shard1) && ri.size() > 0) {
Number sz = ((Number) ri.get(0).getVariable(CORE_IDX.tagName));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index d052d6f..29ccc65 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
@@ -57,6 +58,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
import static org.apache.solr.common.ConditionalMapWriter.dedupeKeyPredicate;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
import static org.apache.solr.common.params.CoreAdminParams.NODE;
import static org.apache.solr.common.util.Utils.handleExp;
import static org.apache.solr.common.util.Utils.time;
@@ -68,7 +70,7 @@ public class PolicyHelper {
private static final String POLICY_MAPPING_KEY = "PolicyHelper.policyMapping";
private static ThreadLocal<Map<String, String>> getPolicyMapping(SolrCloudManager cloudManager) {
- return (ThreadLocal<Map<String, String>>)cloudManager.getObjectCache()
+ return (ThreadLocal<Map<String, String>>) cloudManager.getObjectCache()
.computeIfAbsent(POLICY_MAPPING_KEY, k -> new ThreadLocal<>());
}
@@ -222,43 +224,82 @@ public class PolicyHelper {
.put("config", session.getPolicy());
}
- public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf, SolrCloudManager cloudManager) {
+ public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
+ SolrCloudManager cloudManager) {
+ return getSuggestions(autoScalingConf, cloudManager, 20);
+ }
+
+ public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
+ SolrCloudManager cloudManager, int max) {
Policy policy = autoScalingConf.getPolicy();
- Suggestion.Ctx suggestionCtx = new Suggestion.Ctx();
- suggestionCtx.session = policy.createSession(cloudManager);
- List<Violation> violations = suggestionCtx.session.getViolations();
+ Suggestion.Ctx ctx = new Suggestion.Ctx();
+ ctx.max = max;
+ ctx.session = policy.createSession(cloudManager);
+ List<Violation> violations = ctx.session.getViolations();
for (Violation violation : violations) {
String name = violation.getClause().isPerCollectiontag() ?
violation.getClause().tag.name :
violation.getClause().globalTag.name;
Variable.Type tagType = VariableBase.getTagType(name);
- tagType.getSuggestions(suggestionCtx.setViolation(violation));
- suggestionCtx.violation = null;
+ tagType.getSuggestions(ctx.setViolation(violation));
+ ctx.violation = null;
+ }
+ if (ctx.getSuggestions().size() < max) {
+ suggestOptimizations(ctx);
+ }
+ return ctx.getSuggestions();
+ }
+
+ private static void suggestOptimizations(Suggestion.Ctx ctx) {
+ List<Row> matrix = ctx.session.matrix;
+ if (matrix.isEmpty()) return;
+ for (int i = 0; i < matrix.size(); i++) {
+ Row row = matrix.get(i);
+ Map<String, Collection<String>> collVsShards = new HashMap<>();
+ row.forEachReplica(ri -> collVsShards.computeIfAbsent(ri.getCollection(), s -> new HashSet<>()).add(ri.getShard()));
+ for (Map.Entry<String, Collection<String>> e : collVsShards.entrySet()) {
+ e.setValue(FreeDiskVariable.getSortedShards(Collections.singletonList(row), e.getValue(), e.getKey()));
+ }
+ for (Map.Entry<String, Collection<String>> e : collVsShards.entrySet()) {
+ if (!ctx.needMore()) break;
+ for (String shard : e.getValue()) {
+ if (!ctx.needMore()) break;
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .hint(Hint.COLL_SHARD, new Pair<>(e.getKey(), shard))
+ .hint(Hint.SRC_NODE, row.node);
+ ctx.addSuggestion(suggester);
+ }
+ }
}
- return suggestionCtx.getSuggestions();
}
- /**Use this to dump the state of a system and to generate a testcase
+ /**
+ * Use this to dump the state of a system and to generate a testcase
*/
public static void logState(SolrCloudManager cloudManager, Suggester suggester) {
if (log.isTraceEnabled()) {
- log.trace("LOGSTATE: {}",
- Utils.toJSONString((MapWriter) ew -> {
- ew.put("liveNodes", cloudManager.getClusterStateProvider().getLiveNodes());
- ew.put("suggester", suggester);
- if (suggester.session.nodeStateProvider instanceof MapWriter) {
- MapWriter nodeStateProvider = (MapWriter) suggester.session.nodeStateProvider;
- nodeStateProvider.writeMap(ew);
- }
- try {
- ew.put("autoscalingJson", cloudManager.getDistribStateManager().getAutoScalingConfig());
- } catch (InterruptedException e) {
- }
- }));
+ try {
+ log.trace("LOGSTATE: {}",
+ Utils.writeJson(loggingInfo(cloudManager.getDistribStateManager().getAutoScalingConfig().getPolicy(), cloudManager, suggester),
+ new StringWriter(), true).toString());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
+
+ static MapWriter loggingInfo(Policy policy, SolrCloudManager cloudManager, Suggester suggester) {
+ return ew -> {
+ ew.put("diagnostics", getDiagnostics(policy,
+ cloudManager));
+ if (suggester != null) {
+ ew.put("suggester", suggester);
+ }
+ };
+ }
+
public enum Status {
NULL,
//it is just created and not yet used or all operations on it has been completed fully
@@ -292,7 +333,6 @@ public class PolicyHelper {
/**
* All operations suggested by the current session object
* is complete. Do not even cache anything
- *
*/
private void release(SessionWrapper sessionWrapper) {
synchronized (lockObj) {
@@ -306,7 +346,6 @@ public class PolicyHelper {
/**
* Computing is over for this session and it may contain a new session with new state
* The session can be used by others while the caller is performing operations
- *
*/
private void returnSession(SessionWrapper sessionWrapper) {
TimeSource timeSource = sessionWrapper.session != null ? sessionWrapper.session.cloudManager.getTimeSource() : TimeSource.NANO_TIME;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 85d6f30..dcd5fba 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -154,7 +154,7 @@ public class Row implements MapWriter {
if (row == null) throw new RuntimeException("couldn't get a row");
Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
- String replicaname = "" + new Random().nextInt(1000) + 1000;
+ String replicaname = "SYNTHETIC." + new Random().nextInt(1000) + 1000;
ReplicaInfo ri = new ReplicaInfo(replicaname, replicaname, coll, shard, type, this.node,
Utils.makeMap(ZkStateReader.REPLICA_TYPE, type != null ? type.toString() : Replica.Type.NRT.toString()));
replicas.add(ri);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index db0aab4..9f42b9f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -208,7 +208,7 @@ public abstract class Suggester implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put("type", violation == null ? "improvement" : "violation");
- ew.putIfNotNull("violation",
+ if(violation!= null) ew.put("violation",
new ConditionalMapWriter(violation,
(k, v) -> !"violatingReplicas".equals(k)));
ew.put("operation", operation);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 1f711e5..8f120e2 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -32,6 +32,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.MO
public class Suggestion {
static class Ctx {
+ int max = Integer.MAX_VALUE;
public Policy.Session session;
public Violation violation;
private List<Suggester.SuggestionInfo> suggestions = new ArrayList<>();
@@ -55,6 +56,10 @@ public class Suggestion {
public List<Suggester.SuggestionInfo> getSuggestions() {
return suggestions;
}
+
+ public boolean needMore() {
+ return suggestions.size() < max;
+ }
}
static void suggestNegativeViolations(Suggestion.Ctx ctx, Function<Set<String>, List<String>> shardSorter) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
index 706414d..d351fc2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
@@ -71,7 +71,7 @@ public class ConditionalMapWriter implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- delegate.writeMap(new EntryWriterWrapper(ew));
+ if(delegate!=null) delegate.writeMap(new EntryWriterWrapper(ew));
}
public static BiPredicate<String, Object> dedupeKeyPredicate(Set<String> keys) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index aa8ab38..4bdc00d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -80,6 +80,11 @@ public interface MapWriter extends MapSerializable {
void writeMap(EntryWriter ew) throws IOException;
+
+ default Object _get(String path, Object def) {
+ Object v = Utils.getObjectByPath(this, false, path);
+ return v == null ? def : v;
+ }
/**
* An interface to push one entry at a time to the output.
* The order of the keys is not defined, but we assume they are distinct -- don't call {@code put} more than once
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 9e72607..800c2c1 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -70,6 +70,7 @@ import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableSet;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -267,6 +268,7 @@ public class Utils {
}
public static Object getObjectByPath(Object root, boolean onlyPrimitive, String hierarchy) {
+ if (hierarchy == null) return getObjectByPath(root, onlyPrimitive, singletonList(null));
List<String> parts = StrUtils.splitSmart(hierarchy, '/');
if (parts.get(0).isEmpty()) parts.remove(0);
return getObjectByPath(root, onlyPrimitive, parts);
@@ -362,8 +364,12 @@ public class Utils {
Object val = getVal(obj, s);
if (val == null) return null;
if (idx > -1) {
- List l = (List) val;
- val = idx < l.size() ? l.get(idx) : null;
+ if (val instanceof IteratorWriter) {
+ val = getValueAt((IteratorWriter) val, idx);
+ } else {
+ List l = (List) val;
+ val = idx < l.size() ? l.get(idx) : null;
+ }
}
if (onlyPrimitive && isMapLike(val)) {
return null;
@@ -375,6 +381,27 @@ public class Utils {
return false;
}
+ private static Object getValueAt(IteratorWriter iteratorWriter, int idx) {
+ Object[] result = new Object[1];
+ try {
+ iteratorWriter.writeIter(new IteratorWriter.ItemWriter() {
+ int i = -1;
+
+ @Override
+ public IteratorWriter.ItemWriter add(Object o) {
+ ++i;
+ if (i > idx) return this;
+ if (i == idx) result[0] = o;
+ return this;
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return result[0];
+
+ }
+
private static boolean isMapLike(Object o) {
return o instanceof Map || o instanceof NamedList || o instanceof MapWriter;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json
new file mode 100644
index 0000000..958efc0
--- /dev/null
+++ b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json
@@ -0,0 +1,130 @@
+{
+ "diagnostics":{
+ "sortedNodes":[
+ {
+ "node":"10.0.0.79:7574_solr",
+ "isLive":true,
+ "cores":4.0,
+ "freedisk":140.8341178894043,
+ "totaldisk":233.5667953491211,
+ "replicas":{
+ "gettingstarted":{
+ "shard2":[{
+ "core_node7":{
+ "core":"gettingstarted_shard2_replica_n4",
+ "shard":"shard2",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node3":{
+ "core":"gettingstarted_shard1_replica_n1",
+ "shard":"shard1",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]},
+ "go":{
+ "shard2":[{
+ "core_node7":{
+ "core":"go_shard2_replica_n4",
+ "shard":"shard2",
+ "collection":"go",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node3":{
+ "core":"go_shard1_replica_n1",
+ "shard":"shard1",
+ "collection":"go",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]}}}
+ ,{
+ "node":"10.0.0.79:8984_solr",
+ "isLive":true,
+ "cores":4.0,
+ "freedisk":140.8341178894043,
+ "totaldisk":233.5667953491211,
+ "replicas":{
+ "gettingstarted":{
+ "shard2":[{
+ "core_node8":{
+ "core":"gettingstarted_shard2_replica_n6",
+ "shard":"shard2",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node5":{
+ "core":"gettingstarted_shard1_replica_n2",
+ "shard":"shard1",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]},
+ "go":{
+ "shard2":[{
+ "core_node8":{
+ "core":"go_shard2_replica_n6",
+ "shard":"shard2",
+ "collection":"go",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node5":{
+ "core":"go_shard1_replica_n2",
+ "shard":"shard1",
+ "collection":"go",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]}}}
+ ,{
+ "node":"10.0.0.79:8983_solr",
+ "isLive":true,
+ "cores":0.0,
+ "freedisk":140.8341178894043,
+ "totaldisk":233.5667953491211,
+ "replicas":{}}],
+ "liveNodes":["10.0.0.79:7574_solr",
+ "10.0.0.79:8983_solr",
+ "10.0.0.79:8984_solr"],
+ "violations":[],
+ "config":{
+ "cluster-preferences":[{
+ "minimize":"cores",
+ "precision":1}
+ ,{
+ "maximize":"freedisk"}]}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json
new file mode 100644
index 0000000..ce0b682
--- /dev/null
+++ b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json
@@ -0,0 +1,105 @@
+{"diagnostics":{
+ "sortedNodes":[{
+ "node":"127.0.0.1:63191_solr",
+ "isLive":true,
+ "cores":3.0,
+ "sysprop.zone":"east",
+ "freedisk":1727.1459312438965,
+ "heapUsage":24.97510064011647,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{"zonesTest":{"shard1":[{"core_node5":{
+ "core":"zonesTest_shard1_replica_n2",
+ "leader":"true",
+ "base_url":"https://127.0.0.1:63191/solr",
+ "node_name":"127.0.0.1:63191_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard1",
+ "collection":"zonesTest"}},
+ {"core_node7":{
+ "core":"zonesTest_shard1_replica_n4",
+ "base_url":"https://127.0.0.1:63191/solr",
+ "node_name":"127.0.0.1:63191_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard1",
+ "collection":"zonesTest"}},
+ {"core_node12":{
+ "core":"zonesTest_shard1_replica_n10",
+ "base_url":"https://127.0.0.1:63191/solr",
+ "node_name":"127.0.0.1:63191_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard1",
+ "collection":"zonesTest"}}]}}},
+ {
+ "node":"127.0.0.1:63192_solr",
+ "isLive":true,
+ "cores":3.0,
+ "sysprop.zone":"east",
+ "freedisk":1727.1459312438965,
+ "heapUsage":24.98878807983566,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{"zonesTest":{"shard2":[{"core_node3":{
+ "core":"zonesTest_shard1_replica_n1",
+ "base_url":"https://127.0.0.1:63192/solr",
+ "node_name":"127.0.0.1:63192_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard2",
+ "collection":"zonesTest"}},
+ {"core_node9":{
+ "core":"zonesTest_shard1_replica_n6",
+ "base_url":"https://127.0.0.1:63192/solr",
+ "node_name":"127.0.0.1:63192_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard2",
+ "collection":"zonesTest"}},
+ {"core_node11":{
+ "core":"zonesTest_shard1_replica_n8",
+ "base_url":"https://127.0.0.1:63192/solr",
+ "node_name":"127.0.0.1:63192_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard2",
+ "collection":"zonesTest"}}]}}},
+ {
+ "node":"127.0.0.1:63219_solr",
+ "isLive":true,
+ "cores":0.0,
+ "sysprop.zone":"west",
+ "freedisk":1768.6174201965332,
+ "heapUsage":24.98878807983566,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{}},
+ {
+ "node":"127.0.0.1:63229_solr",
+ "isLive":true,
+ "cores":0.0,
+ "sysprop.zone":"west",
+ "freedisk":1768.6174201965332,
+ "heapUsage":24.98878807983566,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{}}],
+ "liveNodes":["127.0.0.1:63191_solr",
+ "127.0.0.1:63192_solr",
+ "127.0.0.1:63219_solr",
+ "127.0.0.1:63229_solr"]
+}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 4a16259..2b11d0c 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -79,6 +79,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.MO
public class TestPolicy extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
static Suggester createSuggester(SolrCloudManager cloudManager, Map jsonObj, Suggester seed) throws IOException, InterruptedException {
Policy.Session session = null;
if (seed != null) session = seed.session;
@@ -397,19 +398,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
// collect the set of nodes to which replicas are being added
Set<String> nodes = new HashSet<>(2);
- m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(m, true, "operation/path"));
- assertNotNull(Utils.getObjectByPath(m, false, "operation/command/add-replica"));
- nodes.add((String) Utils.getObjectByPath(m, true, "operation/command/add-replica/node"));
+ assertEquals(1.0d, l.get(0)._get("violation/violation/delta", null));
+ assertEquals("POST", l.get(0)._get("operation/method", null));
+ assertEquals("/c/articles_coll/shards", l.get(0)._get("operation/path", null));
+ assertNotNull(l.get(0)._get("operation/command/add-replica", null));
+ nodes.add((String) l.get(0)._get("operation/command/add-replica/node", null));
- m = l.get(1).toMap(new LinkedHashMap<>());
- assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(m, true, "operation/path"));
- assertNotNull(Utils.getObjectByPath(m, false, "operation/command/add-replica"));
- nodes.add((String) Utils.getObjectByPath(m, true, "operation/command/add-replica/node"));
+ assertEquals(1.0d, l.get(1)._get("violation/violation/delta", null));
+ assertEquals("POST", l.get(1)._get("operation/method", null));
+ assertEquals("/c/articles_coll/shards", l.get(1)._get("operation/path", null));
+ assertNotNull(l.get(1)._get("operation/command/add-replica", null));
+ nodes.add((String) l.get(1)._get("operation/command/add-replica/node", null));
assertEquals(2, nodes.size());
assertTrue(nodes.contains("node1"));
@@ -565,18 +564,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
Set<String> targetNodes = new HashSet<>();
Set<String> movedReplicas = new HashSet<>();
for (Suggester.SuggestionInfo suggestionInfo : l) {
- Map s = suggestionInfo.toMap(new LinkedHashMap<>());
- assertEquals("POST", Utils.getObjectByPath(s, true, "operation/method"));
- if (Utils.getObjectByPath(s, false, "operation/command/add-replica") != null) {
+ assertEquals("POST", suggestionInfo._get("operation/method", null));
+ if (suggestionInfo._get("operation/command/add-replica", null) != null) {
numAdds++;
- assertEquals(1.0d, Utils.getObjectByPath(s, true, "violation/violation/delta"));
- assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(s, true, "operation/path"));
- addNodes.add((String) Utils.getObjectByPath(s, true, "operation/command/add-replica/node"));
- } else if (Utils.getObjectByPath(s, false, "operation/command/move-replica") != null) {
+ assertEquals(1.0d, suggestionInfo._get("violation/violation/delta", null));
+ assertEquals("/c/articles_coll/shards", suggestionInfo._get("operation/path", null));
+ addNodes.add((String) suggestionInfo._get("operation/command/add-replica/node", null));
+ } else if (suggestionInfo._get("operation/command/move-replica", null) != null) {
numMoves++;
- assertEquals("/c/articles_coll", Utils.getObjectByPath(s, true, "operation/path"));
- targetNodes.add((String) Utils.getObjectByPath(s, true, "operation/command/move-replica/targetNode"));
- movedReplicas.add((String) Utils.getObjectByPath(s, true, "operation/command/move-replica/replica"));
+ assertEquals("/c/articles_coll", suggestionInfo._get("operation/path", null));
+ targetNodes.add((String) suggestionInfo._get("operation/command/move-replica/targetNode", null));
+ movedReplicas.add((String) suggestionInfo._get("operation/command/move-replica/replica", null));
} else {
fail("Unexpected operation type suggested for suggestion: " + suggestionInfo);
}
@@ -2357,13 +2355,12 @@ public class TestPolicy extends SolrTestCaseJ4 {
cloudManagerWithData(dataproviderdata));
assertFalse(l.isEmpty());
- Map m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/mycoll1", Utils.getObjectByPath(m, true, "operation/path"));
- assertNotNull(Utils.getObjectByPath(m, false, "operation/command/move-replica"));
- assertEquals("10.0.0.6:7574_solr", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
- assertEquals("core_node2", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
+ assertEquals(1.0d, l.get(0)._get( "violation/violation/delta",null));
+ assertEquals("POST", l.get(0)._get("operation/method",null));
+ assertEquals("/c/mycoll1", l.get(0)._get( "operation/path",null));
+ assertNotNull(l.get(0)._get("operation/command/move-replica", null));
+ assertEquals("10.0.0.6:7574_solr", l.get(0)._get( "operation/command/move-replica/targetNode",null));
+ assertEquals("core_node2", l.get(0)._get("operation/command/move-replica/replica", null));
}
@@ -2526,7 +2523,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
- public void testFreeDiskSuggestions() throws IOException {
+ public void testFreeDiskSuggestions() {
String dataproviderdata = "{" +
" liveNodes:[node1,node2]," +
" replicaInfo : {" +
@@ -2557,17 +2554,14 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Suggester.SuggestionInfo> l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(3, l.size());
- Map m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals("r4", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r4", l.get(0)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(0)._get("operation/command/move-replica/targetNode", null));
- m = l.get(1).toMap(new LinkedHashMap<>());
- assertEquals("r3", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r3", l.get(1)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(1)._get("operation/command/move-replica/targetNode", null));
- m = l.get(2).toMap(new LinkedHashMap<>());
- assertEquals("r2", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r2", l.get(2)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(2)._get("operation/command/move-replica/targetNode", null));
autoScalingjson = " { cluster-policy:[" +
@@ -2582,18 +2576,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(0, violations.get(0).getViolatingReplicas().size());
l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
- assertEquals(3, l.size());
- m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals("r4", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals(4, l.size());
+ assertEquals("r4", l.get(0)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(0)._get("operation/command/move-replica/targetNode", null));
+
+ assertEquals("r3", l.get(1)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(1)._get("operation/command/move-replica/targetNode", null));
- m = l.get(1).toMap(new LinkedHashMap<>());
- assertEquals("r3", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r2", l.get(2)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(2)._get("operation/command/move-replica/targetNode", null));
- m = l.get(2).toMap(new LinkedHashMap<>());
- assertEquals("r2", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("improvement", l.get(3)._get("type", null));
}
@@ -2630,10 +2623,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
cloudManagerWithData(dataproviderdata));
assertEquals(2, l.size());
for (Suggester.SuggestionInfo suggestionInfo : l) {
- Map m = suggestionInfo.toMap(new LinkedHashMap<>());
- assertEquals("10.0.0.6:7574_solr", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/mycoll1", Utils.getObjectByPath(m, true, "operation/path"));
+ assertEquals("10.0.0.6:7574_solr", suggestionInfo._get("operation/command/move-replica/targetNode", null));
+ assertEquals("POST", suggestionInfo._get("operation/method", null));
+ assertEquals("/c/mycoll1", suggestionInfo._get("operation/path", null));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f2975c2/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index 71c0287..e902ff9 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -377,6 +377,45 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
}
+ public void testSuggestionsRebalanceOnly() throws IOException {
+ String conf = " {" +
+ " 'cluster-preferences':[{" +
+ " 'minimize':'cores'," +
+ " 'precision':1}," +
+ " {'maximize':'freedisk','precision':100}," +
+ " {'minimize':'sysLoadAvg','precision':10}]," +
+ " 'cluster-policy':[" +
+ "{'replica':'<5','shard':'#EACH','sysprop.zone':['east','west']}]}";
+ Map<String, Object> m = (Map<String, Object>) loadFromResource("testSuggestionsRebalanceOnly.json");
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+ AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(conf));
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics);
+
+ assertEquals(2, suggestions.size());
+ assertEquals("improvement", suggestions.get(0)._get("type",null));
+ assertEquals("127.0.0.1:63229_solr", suggestions.get(0)._get("operation/command/move-replica/targetNode", null));
+ assertEquals("improvement", suggestions.get(1)._get( "type",null));
+ assertEquals("127.0.0.1:63219_solr", suggestions.get(1)._get("operation/command/move-replica/targetNode", null));
+ }
+
+ public void testSuggestionsRebalance2() throws IOException {
+ Map<String, Object> m = (Map<String, Object>) loadFromResource("testSuggestionsRebalance2.json");
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+
+ AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics);
+
+ assertEquals(3, suggestions.size());
+
+ for (Suggester.SuggestionInfo suggestion : suggestions) {
+ assertEquals("improvement", suggestion._get("type", null));
+ assertEquals("10.0.0.79:8983_solr", suggestion._get("operation/command/move-replica/targetNode",null));
+ }
+
+
+
+ }
+
public static Object loadFromResource(String file) throws IOException {
try (InputStream is = TestPolicy2.class.getResourceAsStream("/solrj/solr/autoscaling/" + file)) {
return Utils.fromJSON(is);