You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2018/09/24 14:39:06 UTC
[1/2] lucene-solr:branch_7x: SOLR-11522: /autoscaling/suggestions now
include rebalance options as well even if there are no violations
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 503726f4f -> 0f9a8a818
SOLR-11522: /autoscaling/suggestions now include rebalance options as well even if there are no violations
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4fa89330
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4fa89330
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4fa89330
Branch: refs/heads/branch_7x
Commit: 4fa893304ae07db9fc229690cc0481f242e4053a
Parents: 503726f
Author: noble <no...@apache.org>
Authored: Tue Sep 25 00:34:21 2018 +1000
Committer: noble <no...@apache.org>
Committed: Tue Sep 25 00:38:44 2018 +1000
----------------------------------------------------------------------
.../cloud/autoscaling/FreeDiskVariable.java | 12 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 87 +++++++++----
.../client/solrj/cloud/autoscaling/Row.java | 2 +-
.../solrj/cloud/autoscaling/Suggester.java | 2 +-
.../solrj/cloud/autoscaling/Suggestion.java | 5 +
.../solr/common/ConditionalMapWriter.java | 2 +-
.../java/org/apache/solr/common/MapWriter.java | 5 +
.../java/org/apache/solr/common/util/Utils.java | 31 ++++-
.../autoscaling/testSuggestionsRebalance2.json | 130 +++++++++++++++++++
.../testSuggestionsRebalanceOnly.json | 105 +++++++++++++++
.../solrj/cloud/autoscaling/TestPolicy.java | 98 +++++++-------
.../solrj/cloud/autoscaling/TestPolicy2.java | 39 ++++++
12 files changed, 429 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
index 600695a..778a837 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
@@ -18,9 +18,9 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Comparator;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -104,19 +104,17 @@ public class FreeDiskVariable extends VariableBase {
}
}
} else if (ctx.violation.replicaCountDelta < 0) {
- suggestNegativeViolations(ctx, shards -> getSortedShards(ctx,shards));
+ suggestNegativeViolations(ctx, shards -> getSortedShards(ctx.session.matrix, shards, ctx.violation.coll));
}
}
-
-
- private List<String> getSortedShards(Suggestion.Ctx ctx, Set<String> shardSet) {
+ static List<String> getSortedShards(List<Row> matrix, Collection<String> shardSet, String coll) {
return shardSet.stream()
.map(shard1 -> {
AtomicReference<Pair<String, Long>> result = new AtomicReference<>();
- for (Row node : ctx.session.matrix) {
- node.forEachShard(ctx.violation.coll, (s, ri) -> {
+ for (Row node : matrix) {
+ node.forEachShard(coll, (s, ri) -> {
if (result.get() != null) return;
if (s.equals(shard1) && ri.size() > 0) {
Number sz = ((Number) ri.get(0).getVariable(CORE_IDX.tagName));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index fa2f166..7a1a727 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
@@ -57,6 +58,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
import static org.apache.solr.common.ConditionalMapWriter.dedupeKeyPredicate;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
import static org.apache.solr.common.params.CoreAdminParams.NODE;
import static org.apache.solr.common.util.Utils.handleExp;
import static org.apache.solr.common.util.Utils.time;
@@ -68,7 +70,7 @@ public class PolicyHelper {
private static final String POLICY_MAPPING_KEY = "PolicyHelper.policyMapping";
private static ThreadLocal<Map<String, String>> getPolicyMapping(SolrCloudManager cloudManager) {
- return (ThreadLocal<Map<String, String>>)cloudManager.getObjectCache()
+ return (ThreadLocal<Map<String, String>>) cloudManager.getObjectCache()
.computeIfAbsent(POLICY_MAPPING_KEY, k -> new ThreadLocal<>());
}
@@ -222,43 +224,82 @@ public class PolicyHelper {
.put("config", session.getPolicy());
}
- public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf, SolrCloudManager cloudManager) {
+ public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
+ SolrCloudManager cloudManager) {
+ return getSuggestions(autoScalingConf, cloudManager, 20);
+ }
+
+ public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
+ SolrCloudManager cloudManager, int max) {
Policy policy = autoScalingConf.getPolicy();
- Suggestion.Ctx suggestionCtx = new Suggestion.Ctx();
- suggestionCtx.session = policy.createSession(cloudManager);
- List<Violation> violations = suggestionCtx.session.getViolations();
+ Suggestion.Ctx ctx = new Suggestion.Ctx();
+ ctx.max = max;
+ ctx.session = policy.createSession(cloudManager);
+ List<Violation> violations = ctx.session.getViolations();
for (Violation violation : violations) {
String name = violation.getClause().isPerCollectiontag() ?
violation.getClause().tag.name :
violation.getClause().globalTag.name;
Variable.Type tagType = VariableBase.getTagType(name);
- tagType.getSuggestions(suggestionCtx.setViolation(violation));
- suggestionCtx.violation = null;
+ tagType.getSuggestions(ctx.setViolation(violation));
+ ctx.violation = null;
+ }
+ if (ctx.getSuggestions().size() < max) {
+ suggestOptimizations(ctx);
+ }
+ return ctx.getSuggestions();
+ }
+
+ private static void suggestOptimizations(Suggestion.Ctx ctx) {
+ List<Row> matrix = ctx.session.matrix;
+ if (matrix.isEmpty()) return;
+ for (int i = 0; i < matrix.size(); i++) {
+ Row row = matrix.get(i);
+ Map<String, Collection<String>> collVsShards = new HashMap<>();
+ row.forEachReplica(ri -> collVsShards.computeIfAbsent(ri.getCollection(), s -> new HashSet<>()).add(ri.getShard()));
+ for (Map.Entry<String, Collection<String>> e : collVsShards.entrySet()) {
+ e.setValue(FreeDiskVariable.getSortedShards(Collections.singletonList(row), e.getValue(), e.getKey()));
+ }
+ for (Map.Entry<String, Collection<String>> e : collVsShards.entrySet()) {
+ if (!ctx.needMore()) break;
+ for (String shard : e.getValue()) {
+ if (!ctx.needMore()) break;
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .hint(Hint.COLL_SHARD, new Pair<>(e.getKey(), shard))
+ .hint(Hint.SRC_NODE, row.node);
+ ctx.addSuggestion(suggester);
+ }
+ }
}
- return suggestionCtx.getSuggestions();
}
- /**Use this to dump the state of a system and to generate a testcase
+ /**
+ * Use this to dump the state of a system and to generate a testcase
*/
public static void logState(SolrCloudManager cloudManager, Suggester suggester) {
if (log.isTraceEnabled()) {
- log.trace("LOGSTATE: {}",
- Utils.toJSONString((MapWriter) ew -> {
- ew.put("liveNodes", cloudManager.getClusterStateProvider().getLiveNodes());
- ew.put("suggester", suggester);
- if (suggester.session.nodeStateProvider instanceof MapWriter) {
- MapWriter nodeStateProvider = (MapWriter) suggester.session.nodeStateProvider;
- nodeStateProvider.writeMap(ew);
- }
- try {
- ew.put("autoscalingJson", cloudManager.getDistribStateManager().getAutoScalingConfig());
- } catch (InterruptedException e) {
- }
- }));
+ try {
+ log.trace("LOGSTATE: {}",
+ Utils.writeJson(loggingInfo(cloudManager.getDistribStateManager().getAutoScalingConfig().getPolicy(), cloudManager, suggester),
+ new StringWriter(), true).toString());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
+
+ static MapWriter loggingInfo(Policy policy, SolrCloudManager cloudManager, Suggester suggester) {
+ return ew -> {
+ ew.put("diagnostics", getDiagnostics(policy,
+ cloudManager));
+ if (suggester != null) {
+ ew.put("suggester", suggester);
+ }
+ };
+ }
+
public enum Status {
NULL,
//it is just created and not yet used or all operations on it has been completed fully
@@ -292,7 +333,6 @@ public class PolicyHelper {
/**
* All operations suggested by the current session object
* is complete. Do not even cache anything
- *
*/
private void release(SessionWrapper sessionWrapper) {
synchronized (lockObj) {
@@ -306,7 +346,6 @@ public class PolicyHelper {
/**
* Computing is over for this session and it may contain a new session with new state
* The session can be used by others while the caller is performing operations
- *
*/
private void returnSession(SessionWrapper sessionWrapper) {
TimeSource timeSource = sessionWrapper.session != null ? sessionWrapper.session.cloudManager.getTimeSource() : TimeSource.NANO_TIME;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 85d6f30..dcd5fba 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -154,7 +154,7 @@ public class Row implements MapWriter {
if (row == null) throw new RuntimeException("couldn't get a row");
Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
- String replicaname = "" + new Random().nextInt(1000) + 1000;
+ String replicaname = "SYNTHETIC." + new Random().nextInt(1000) + 1000;
ReplicaInfo ri = new ReplicaInfo(replicaname, replicaname, coll, shard, type, this.node,
Utils.makeMap(ZkStateReader.REPLICA_TYPE, type != null ? type.toString() : Replica.Type.NRT.toString()));
replicas.add(ri);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index db0aab4..9f42b9f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -208,7 +208,7 @@ public abstract class Suggester implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put("type", violation == null ? "improvement" : "violation");
- ew.putIfNotNull("violation",
+ if(violation!= null) ew.put("violation",
new ConditionalMapWriter(violation,
(k, v) -> !"violatingReplicas".equals(k)));
ew.put("operation", operation);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 1f711e5..8f120e2 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -32,6 +32,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.MO
public class Suggestion {
static class Ctx {
+ int max = Integer.MAX_VALUE;
public Policy.Session session;
public Violation violation;
private List<Suggester.SuggestionInfo> suggestions = new ArrayList<>();
@@ -55,6 +56,10 @@ public class Suggestion {
public List<Suggester.SuggestionInfo> getSuggestions() {
return suggestions;
}
+
+ public boolean needMore() {
+ return suggestions.size() < max;
+ }
}
static void suggestNegativeViolations(Suggestion.Ctx ctx, Function<Set<String>, List<String>> shardSorter) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
index 706414d..d351fc2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
@@ -71,7 +71,7 @@ public class ConditionalMapWriter implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- delegate.writeMap(new EntryWriterWrapper(ew));
+ if(delegate!=null) delegate.writeMap(new EntryWriterWrapper(ew));
}
public static BiPredicate<String, Object> dedupeKeyPredicate(Set<String> keys) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index aa8ab38..4bdc00d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -80,6 +80,11 @@ public interface MapWriter extends MapSerializable {
void writeMap(EntryWriter ew) throws IOException;
+
+ default Object _get(String path, Object def) {
+ Object v = Utils.getObjectByPath(this, false, path);
+ return v == null ? def : v;
+ }
/**
* An interface to push one entry at a time to the output.
* The order of the keys is not defined, but we assume they are distinct -- don't call {@code put} more than once
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 9e72607..800c2c1 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -70,6 +70,7 @@ import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static java.util.Collections.unmodifiableSet;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -267,6 +268,7 @@ public class Utils {
}
public static Object getObjectByPath(Object root, boolean onlyPrimitive, String hierarchy) {
+ if (hierarchy == null) return getObjectByPath(root, onlyPrimitive, singletonList(null));
List<String> parts = StrUtils.splitSmart(hierarchy, '/');
if (parts.get(0).isEmpty()) parts.remove(0);
return getObjectByPath(root, onlyPrimitive, parts);
@@ -362,8 +364,12 @@ public class Utils {
Object val = getVal(obj, s);
if (val == null) return null;
if (idx > -1) {
- List l = (List) val;
- val = idx < l.size() ? l.get(idx) : null;
+ if (val instanceof IteratorWriter) {
+ val = getValueAt((IteratorWriter) val, idx);
+ } else {
+ List l = (List) val;
+ val = idx < l.size() ? l.get(idx) : null;
+ }
}
if (onlyPrimitive && isMapLike(val)) {
return null;
@@ -375,6 +381,27 @@ public class Utils {
return false;
}
+ private static Object getValueAt(IteratorWriter iteratorWriter, int idx) {
+ Object[] result = new Object[1];
+ try {
+ iteratorWriter.writeIter(new IteratorWriter.ItemWriter() {
+ int i = -1;
+
+ @Override
+ public IteratorWriter.ItemWriter add(Object o) {
+ ++i;
+ if (i > idx) return this;
+ if (i == idx) result[0] = o;
+ return this;
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return result[0];
+
+ }
+
private static boolean isMapLike(Object o) {
return o instanceof Map || o instanceof NamedList || o instanceof MapWriter;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json
new file mode 100644
index 0000000..958efc0
--- /dev/null
+++ b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json
@@ -0,0 +1,130 @@
+{
+ "diagnostics":{
+ "sortedNodes":[
+ {
+ "node":"10.0.0.79:7574_solr",
+ "isLive":true,
+ "cores":4.0,
+ "freedisk":140.8341178894043,
+ "totaldisk":233.5667953491211,
+ "replicas":{
+ "gettingstarted":{
+ "shard2":[{
+ "core_node7":{
+ "core":"gettingstarted_shard2_replica_n4",
+ "shard":"shard2",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node3":{
+ "core":"gettingstarted_shard1_replica_n1",
+ "shard":"shard1",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]},
+ "go":{
+ "shard2":[{
+ "core_node7":{
+ "core":"go_shard2_replica_n4",
+ "shard":"shard2",
+ "collection":"go",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node3":{
+ "core":"go_shard1_replica_n1",
+ "shard":"shard1",
+ "collection":"go",
+ "node_name":"10.0.0.79:7574_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.79:7574/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]}}}
+ ,{
+ "node":"10.0.0.79:8984_solr",
+ "isLive":true,
+ "cores":4.0,
+ "freedisk":140.8341178894043,
+ "totaldisk":233.5667953491211,
+ "replicas":{
+ "gettingstarted":{
+ "shard2":[{
+ "core_node8":{
+ "core":"gettingstarted_shard2_replica_n6",
+ "shard":"shard2",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node5":{
+ "core":"gettingstarted_shard1_replica_n2",
+ "shard":"shard1",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]},
+ "go":{
+ "shard2":[{
+ "core_node8":{
+ "core":"go_shard2_replica_n6",
+ "shard":"shard2",
+ "collection":"go",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}],
+ "shard1":[{
+ "core_node5":{
+ "core":"go_shard1_replica_n2",
+ "shard":"shard1",
+ "collection":"go",
+ "node_name":"10.0.0.79:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.79:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]}}}
+ ,{
+ "node":"10.0.0.79:8983_solr",
+ "isLive":true,
+ "cores":0.0,
+ "freedisk":140.8341178894043,
+ "totaldisk":233.5667953491211,
+ "replicas":{}}],
+ "liveNodes":["10.0.0.79:7574_solr",
+ "10.0.0.79:8983_solr",
+ "10.0.0.79:8984_solr"],
+ "violations":[],
+ "config":{
+ "cluster-preferences":[{
+ "minimize":"cores",
+ "precision":1}
+ ,{
+ "maximize":"freedisk"}]}}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json
new file mode 100644
index 0000000..ce0b682
--- /dev/null
+++ b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json
@@ -0,0 +1,105 @@
+{"diagnostics":{
+ "sortedNodes":[{
+ "node":"127.0.0.1:63191_solr",
+ "isLive":true,
+ "cores":3.0,
+ "sysprop.zone":"east",
+ "freedisk":1727.1459312438965,
+ "heapUsage":24.97510064011647,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{"zonesTest":{"shard1":[{"core_node5":{
+ "core":"zonesTest_shard1_replica_n2",
+ "leader":"true",
+ "base_url":"https://127.0.0.1:63191/solr",
+ "node_name":"127.0.0.1:63191_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard1",
+ "collection":"zonesTest"}},
+ {"core_node7":{
+ "core":"zonesTest_shard1_replica_n4",
+ "base_url":"https://127.0.0.1:63191/solr",
+ "node_name":"127.0.0.1:63191_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard1",
+ "collection":"zonesTest"}},
+ {"core_node12":{
+ "core":"zonesTest_shard1_replica_n10",
+ "base_url":"https://127.0.0.1:63191/solr",
+ "node_name":"127.0.0.1:63191_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard1",
+ "collection":"zonesTest"}}]}}},
+ {
+ "node":"127.0.0.1:63192_solr",
+ "isLive":true,
+ "cores":3.0,
+ "sysprop.zone":"east",
+ "freedisk":1727.1459312438965,
+ "heapUsage":24.98878807983566,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{"zonesTest":{"shard2":[{"core_node3":{
+ "core":"zonesTest_shard1_replica_n1",
+ "base_url":"https://127.0.0.1:63192/solr",
+ "node_name":"127.0.0.1:63192_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard2",
+ "collection":"zonesTest"}},
+ {"core_node9":{
+ "core":"zonesTest_shard1_replica_n6",
+ "base_url":"https://127.0.0.1:63192/solr",
+ "node_name":"127.0.0.1:63192_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard2",
+ "collection":"zonesTest"}},
+ {"core_node11":{
+ "core":"zonesTest_shard1_replica_n8",
+ "base_url":"https://127.0.0.1:63192/solr",
+ "node_name":"127.0.0.1:63192_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8,
+ "shard":"shard2",
+ "collection":"zonesTest"}}]}}},
+ {
+ "node":"127.0.0.1:63219_solr",
+ "isLive":true,
+ "cores":0.0,
+ "sysprop.zone":"west",
+ "freedisk":1768.6174201965332,
+ "heapUsage":24.98878807983566,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{}},
+ {
+ "node":"127.0.0.1:63229_solr",
+ "isLive":true,
+ "cores":0.0,
+ "sysprop.zone":"west",
+ "freedisk":1768.6174201965332,
+ "heapUsage":24.98878807983566,
+ "sysLoadAvg":272.75390625,
+ "totaldisk":1037.938980102539,
+ "replicas":{}}],
+ "liveNodes":["127.0.0.1:63191_solr",
+ "127.0.0.1:63192_solr",
+ "127.0.0.1:63219_solr",
+ "127.0.0.1:63229_solr"]
+}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 4a16259..2b11d0c 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -79,6 +79,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.MO
public class TestPolicy extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
static Suggester createSuggester(SolrCloudManager cloudManager, Map jsonObj, Suggester seed) throws IOException, InterruptedException {
Policy.Session session = null;
if (seed != null) session = seed.session;
@@ -397,19 +398,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
// collect the set of nodes to which replicas are being added
Set<String> nodes = new HashSet<>(2);
- m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(m, true, "operation/path"));
- assertNotNull(Utils.getObjectByPath(m, false, "operation/command/add-replica"));
- nodes.add((String) Utils.getObjectByPath(m, true, "operation/command/add-replica/node"));
+ assertEquals(1.0d, l.get(0)._get("violation/violation/delta", null));
+ assertEquals("POST", l.get(0)._get("operation/method", null));
+ assertEquals("/c/articles_coll/shards", l.get(0)._get("operation/path", null));
+ assertNotNull(l.get(0)._get("operation/command/add-replica", null));
+ nodes.add((String) l.get(0)._get("operation/command/add-replica/node", null));
- m = l.get(1).toMap(new LinkedHashMap<>());
- assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(m, true, "operation/path"));
- assertNotNull(Utils.getObjectByPath(m, false, "operation/command/add-replica"));
- nodes.add((String) Utils.getObjectByPath(m, true, "operation/command/add-replica/node"));
+ assertEquals(1.0d, l.get(1)._get("violation/violation/delta", null));
+ assertEquals("POST", l.get(1)._get("operation/method", null));
+ assertEquals("/c/articles_coll/shards", l.get(1)._get("operation/path", null));
+ assertNotNull(l.get(1)._get("operation/command/add-replica", null));
+ nodes.add((String) l.get(1)._get("operation/command/add-replica/node", null));
assertEquals(2, nodes.size());
assertTrue(nodes.contains("node1"));
@@ -565,18 +564,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
Set<String> targetNodes = new HashSet<>();
Set<String> movedReplicas = new HashSet<>();
for (Suggester.SuggestionInfo suggestionInfo : l) {
- Map s = suggestionInfo.toMap(new LinkedHashMap<>());
- assertEquals("POST", Utils.getObjectByPath(s, true, "operation/method"));
- if (Utils.getObjectByPath(s, false, "operation/command/add-replica") != null) {
+ assertEquals("POST", suggestionInfo._get("operation/method", null));
+ if (suggestionInfo._get("operation/command/add-replica", null) != null) {
numAdds++;
- assertEquals(1.0d, Utils.getObjectByPath(s, true, "violation/violation/delta"));
- assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(s, true, "operation/path"));
- addNodes.add((String) Utils.getObjectByPath(s, true, "operation/command/add-replica/node"));
- } else if (Utils.getObjectByPath(s, false, "operation/command/move-replica") != null) {
+ assertEquals(1.0d, suggestionInfo._get("violation/violation/delta", null));
+ assertEquals("/c/articles_coll/shards", suggestionInfo._get("operation/path", null));
+ addNodes.add((String) suggestionInfo._get("operation/command/add-replica/node", null));
+ } else if (suggestionInfo._get("operation/command/move-replica", null) != null) {
numMoves++;
- assertEquals("/c/articles_coll", Utils.getObjectByPath(s, true, "operation/path"));
- targetNodes.add((String) Utils.getObjectByPath(s, true, "operation/command/move-replica/targetNode"));
- movedReplicas.add((String) Utils.getObjectByPath(s, true, "operation/command/move-replica/replica"));
+ assertEquals("/c/articles_coll", suggestionInfo._get("operation/path", null));
+ targetNodes.add((String) suggestionInfo._get("operation/command/move-replica/targetNode", null));
+ movedReplicas.add((String) suggestionInfo._get("operation/command/move-replica/replica", null));
} else {
fail("Unexpected operation type suggested for suggestion: " + suggestionInfo);
}
@@ -2357,13 +2355,12 @@ public class TestPolicy extends SolrTestCaseJ4 {
cloudManagerWithData(dataproviderdata));
assertFalse(l.isEmpty());
- Map m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/mycoll1", Utils.getObjectByPath(m, true, "operation/path"));
- assertNotNull(Utils.getObjectByPath(m, false, "operation/command/move-replica"));
- assertEquals("10.0.0.6:7574_solr", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
- assertEquals("core_node2", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
+ assertEquals(1.0d, l.get(0)._get( "violation/violation/delta",null));
+ assertEquals("POST", l.get(0)._get("operation/method",null));
+ assertEquals("/c/mycoll1", l.get(0)._get( "operation/path",null));
+ assertNotNull(l.get(0)._get("operation/command/move-replica", null));
+ assertEquals("10.0.0.6:7574_solr", l.get(0)._get( "operation/command/move-replica/targetNode",null));
+ assertEquals("core_node2", l.get(0)._get("operation/command/move-replica/replica", null));
}
@@ -2526,7 +2523,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
- public void testFreeDiskSuggestions() throws IOException {
+ public void testFreeDiskSuggestions() {
String dataproviderdata = "{" +
" liveNodes:[node1,node2]," +
" replicaInfo : {" +
@@ -2557,17 +2554,14 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Suggester.SuggestionInfo> l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(3, l.size());
- Map m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals("r4", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r4", l.get(0)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(0)._get("operation/command/move-replica/targetNode", null));
- m = l.get(1).toMap(new LinkedHashMap<>());
- assertEquals("r3", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r3", l.get(1)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(1)._get("operation/command/move-replica/targetNode", null));
- m = l.get(2).toMap(new LinkedHashMap<>());
- assertEquals("r2", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r2", l.get(2)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(2)._get("operation/command/move-replica/targetNode", null));
autoScalingjson = " { cluster-policy:[" +
@@ -2582,18 +2576,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(0, violations.get(0).getViolatingReplicas().size());
l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
- assertEquals(3, l.size());
- m = l.get(0).toMap(new LinkedHashMap<>());
- assertEquals("r4", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals(4, l.size());
+ assertEquals("r4", l.get(0)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(0)._get("operation/command/move-replica/targetNode", null));
+
+ assertEquals("r3", l.get(1)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(1)._get("operation/command/move-replica/targetNode", null));
- m = l.get(1).toMap(new LinkedHashMap<>());
- assertEquals("r3", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("r2", l.get(2)._get("operation/command/move-replica/replica", null));
+ assertEquals("node1", l.get(2)._get("operation/command/move-replica/targetNode", null));
- m = l.get(2).toMap(new LinkedHashMap<>());
- assertEquals("r2", Utils.getObjectByPath(m, true, "operation/command/move-replica/replica"));
- assertEquals("node1", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
+ assertEquals("improvement", l.get(3)._get("type", null));
}
@@ -2630,10 +2623,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
cloudManagerWithData(dataproviderdata));
assertEquals(2, l.size());
for (Suggester.SuggestionInfo suggestionInfo : l) {
- Map m = suggestionInfo.toMap(new LinkedHashMap<>());
- assertEquals("10.0.0.6:7574_solr", Utils.getObjectByPath(m, true, "operation/command/move-replica/targetNode"));
- assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
- assertEquals("/c/mycoll1", Utils.getObjectByPath(m, true, "operation/path"));
+ assertEquals("10.0.0.6:7574_solr", suggestionInfo._get("operation/command/move-replica/targetNode", null));
+ assertEquals("POST", suggestionInfo._get("operation/method", null));
+ assertEquals("/c/mycoll1", suggestionInfo._get("operation/path", null));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa89330/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index 71c0287..e902ff9 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -377,6 +377,45 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
}
+ public void testSuggestionsRebalanceOnly() throws IOException {
+ String conf = " {" +
+ " 'cluster-preferences':[{" +
+ " 'minimize':'cores'," +
+ " 'precision':1}," +
+ " {'maximize':'freedisk','precision':100}," +
+ " {'minimize':'sysLoadAvg','precision':10}]," +
+ " 'cluster-policy':[" +
+ "{'replica':'<5','shard':'#EACH','sysprop.zone':['east','west']}]}";
+ Map<String, Object> m = (Map<String, Object>) loadFromResource("testSuggestionsRebalanceOnly.json");
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+ AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(conf));
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics);
+
+ assertEquals(2, suggestions.size());
+ assertEquals("improvement", suggestions.get(0)._get("type",null));
+ assertEquals("127.0.0.1:63229_solr", suggestions.get(0)._get("operation/command/move-replica/targetNode", null));
+ assertEquals("improvement", suggestions.get(1)._get( "type",null));
+ assertEquals("127.0.0.1:63219_solr", suggestions.get(1)._get("operation/command/move-replica/targetNode", null));
+ }
+
+ public void testSuggestionsRebalance2() throws IOException {
+ Map<String, Object> m = (Map<String, Object>) loadFromResource("testSuggestionsRebalance2.json");
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+
+ AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics);
+
+ assertEquals(3, suggestions.size());
+
+ for (Suggester.SuggestionInfo suggestion : suggestions) {
+ assertEquals("improvement", suggestion._get("type", null));
+ assertEquals("10.0.0.79:8983_solr", suggestion._get("operation/command/move-replica/targetNode",null));
+ }
+
+
+
+ }
+
public static Object loadFromResource(String file) throws IOException {
try (InputStream is = TestPolicy2.class.getResourceAsStream("/solrj/solr/autoscaling/" + file)) {
return Utils.fromJSON(is);
[2/2] lucene-solr:branch_7x: SOLR-11522: /autoscaling/suggestions now
include rebalance options as well even if there are no violations
Posted by no...@apache.org.
SOLR-11522: /autoscaling/suggestions now include rebalance options as well even if there are no violations
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0f9a8a81
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0f9a8a81
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0f9a8a81
Branch: refs/heads/branch_7x
Commit: 0f9a8a818a3cd07b10fbec9f279538e828d816c8
Parents: 4fa8933
Author: noble <no...@apache.org>
Authored: Tue Sep 25 00:38:54 2018 +1000
Committer: noble <no...@apache.org>
Committed: Tue Sep 25 00:38:54 2018 +1000
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f9a8a81/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index bcdf4bf..bb0846c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -35,6 +35,9 @@ New Features
error. Previously, the collapsing behavior was unreliable and undefined despite no explicit error.
(Munendra S N, David Smiley)
+* SOLR-11522: /autoscaling/suggestions now include rebalance options as well even if there are no violations (noble)
+
+
Other Changes
----------------------