You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2018/09/10 02:25:47 UTC
lucene-solr:branch_7x: SOLR-12738: Incorrect Suggestions in
autoscaling framework and refactoring
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x ba1f6c804 -> 6cdcf09f3
SOLR-12738: Incorrect Suggestions in autoscaling framework and refactoring
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6cdcf09f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6cdcf09f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6cdcf09f
Branch: refs/heads/branch_7x
Commit: 6cdcf09f345bd01f894e3530d1cfb783da76a4c6
Parents: ba1f6c8
Author: Noble Paul <no...@apache.org>
Authored: Mon Sep 10 12:24:24 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Mon Sep 10 12:25:29 2018 +1000
----------------------------------------------------------------------
.../client/solrj/cloud/autoscaling/Clause.java | 252 ++++++++++++-------
.../solrj/cloud/autoscaling/Condition.java | 18 +-
.../solrj/cloud/autoscaling/CoresVariable.java | 5 +-
.../cloud/autoscaling/FreeDiskVariable.java | 40 ++-
.../client/solrj/cloud/autoscaling/Operand.java | 1 +
.../solrj/cloud/autoscaling/ReplicaCount.java | 48 +++-
.../client/solrj/cloud/autoscaling/Row.java | 35 ++-
.../solrj/cloud/autoscaling/Suggestion.java | 43 +++-
.../solrj/cloud/autoscaling/Variable.java | 18 +-
.../solrj/cloud/autoscaling/VariableBase.java | 16 +-
.../solrj/cloud/autoscaling/Violation.java | 31 +--
.../autoscaling/WithCollectionVariable.java | 3 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 25 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 193 +++++++++++++-
14 files changed, 537 insertions(+), 191 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index 3a57a1f..820f335 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -20,7 +20,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
@@ -46,6 +46,7 @@ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.RANGE_EQUAL
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
+import static org.apache.solr.common.params.CoreAdminParams.NODE;
import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
import static org.apache.solr.common.params.CoreAdminParams.SHARD;
import static org.apache.solr.common.util.StrUtils.formatString;
@@ -168,7 +169,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
return globalTag;
}
- private Condition evaluateValue(Condition condition, Function<Condition, Object> computedValueEvaluator) {
+ Condition evaluateValue(Condition condition, Function<Condition, Object> computedValueEvaluator) {
if (condition == null) return null;
if (condition.computedType == null) return condition;
Object val = computedValueEvaluator.apply(condition);
@@ -358,107 +359,184 @@ public class Clause implements MapWriter, Comparable<Clause> {
return operand;
}
- public List<Violation> test(Policy.Session session, double[] deviations) {
- ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
- Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator);
- if (isPerCollectiontag()) {
- Map<String, Map<String, Map<String, ReplicaCount>>> replicaCounts = computeReplicaCounts(session.matrix, computedValueEvaluator);
- for (Map.Entry<String, Map<String, Map<String, ReplicaCount>>> e : replicaCounts.entrySet()) {
- computedValueEvaluator.collName = e.getKey();
- if (!collection.isPass(computedValueEvaluator.collName)) continue;
- for (Map.Entry<String, Map<String, ReplicaCount>> shardVsCount : e.getValue().entrySet()) {
- computedValueEvaluator.shardName = shardVsCount.getKey();
- if (!shard.isPass(computedValueEvaluator.shardName)) continue;
- for (Map.Entry<String, ReplicaCount> counts : shardVsCount.getValue().entrySet()) {
- if (tag.varType.meta.isNodeSpecificVal()) computedValueEvaluator.node = counts.getKey();
- SealedClause sealedClause = getSealedClause(computedValueEvaluator);
- ReplicaCount replicas = counts.getValue();
- if (!sealedClause.replica.isPass(replicas)) {
- Violation violation = new Violation(sealedClause,
- computedValueEvaluator.collName,
- computedValueEvaluator.shardName,
- tag.varType.meta.isNodeSpecificVal() ? computedValueEvaluator.node : null,
- counts.getValue(),
- sealedClause.getReplica().delta(replicas),
- tag.varType.meta.isNodeSpecificVal() ? null : counts.getKey());
- tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
- } else {
- if (deviations != null && sealedClause.replica.op == RANGE_EQUAL) {
- Number actualCount = replicas.getVal(type);
- Double realDelta = ((RangeVal) sealedClause.replica.val).realDelta(actualCount.doubleValue());
- realDelta = this.isReplicaZero() ? -1 * realDelta : realDelta;
- deviations[0] += Math.abs(realDelta);
- }
- }
- }
+ List<Violation> testGroupNodes(Policy.Session session, double[] deviations) {
+ //e.g: {replica:'#EQUAL', shard:'#EACH', sysprop.zone:'#EACH'}
+ ComputedValueEvaluator eval = new ComputedValueEvaluator(session);
+ eval.collName = (String) collection.getValue();
+ Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, eval);
+
+ Set tags = new HashSet();
+ for (Row row : session.matrix) {
+ eval.node = row.node;
+ Condition tag = this.tag;
+ if (tag.computedType != null) tag = evaluateValue(tag, eval);
+ Object val = row.getVal(tag.name);
+ if (val != null && tag.isPass(val)) {
+ if (tag.op == LESS_THAN || tag.op == GREATER_THAN) {
+ tags.add(this.tag);
+ } else {
+ tags.add(val);
}
}
- } else {
- for (Row r : session.matrix) {
- computedValueEvaluator.node = r.node;
- SealedClause sealedClause = getSealedClause(computedValueEvaluator);
- if (!sealedClause.getGlobalTag().isPass(r)) {
- sealedClause.getGlobalTag().varType.addViolatingReplicas(ctx.reset(null, null,
- new Violation(sealedClause, null, null, r.node, r.getVal(sealedClause.globalTag.name),
- sealedClause.globalTag.delta(r.getVal(globalTag.name)), null)));
+ }
+ if (tags.isEmpty()) return Collections.emptyList();
+
+ Set<String> shards = getShardNames(session, eval);
+
+ for (String s : shards) {
+ final ReplicaCount replicaCount = new ReplicaCount();
+ eval.shardName = s;
+
+ for (Object t : tags) {
+ replicaCount.reset();
+ for (Row row : session.matrix) {
+ eval.node = row.node;
+ if (t instanceof Condition) {
+ Condition tag = (Condition) t;
+ if (tag.computedType != null) tag = evaluateValue(tag, eval);
+ if (!tag.isPass(row)) continue;
+ } else {
+ if (!t.equals(row.getVal(tag.name))) continue;
+ }
+ addReplicaCountsForNode(eval, replicaCount, row);
+ }
+
+ SealedClause sealedClause = this.getSealedClause(eval);
+ if (!sealedClause.replica.isPass(replicaCount)) {
+ ReplicaCount replicaCountCopy = replicaCount.copy();
+ Violation violation = new Violation(sealedClause,
+ eval.collName,
+ eval.shardName,
+ null,
+ replicaCountCopy,
+ sealedClause.getReplica().replicaCountDelta(replicaCountCopy),
+ t);
+ ctx.resetAndAddViolation(t, replicaCountCopy, violation);
+ sealedClause.addViolatingReplicas(sealedClause.tag, eval, ctx, tag.name, t, violation, session);
+ } else {
+ computeDeviation(deviations, replicaCount, sealedClause);
}
}
}
return ctx.allViolations;
+ }
+ private void computeDeviation(double[] deviations, ReplicaCount replicaCount, SealedClause sealedClause) {
+ if (deviations != null && sealedClause.replica.op == RANGE_EQUAL) {
+ Number actualCount = replicaCount.getVal(type);
+ Double realDelta = ((RangeVal) sealedClause.replica.val).realDelta(actualCount.doubleValue());
+ realDelta = this.isReplicaZero() ? -1 * realDelta : realDelta;
+ deviations[0] += Math.abs(realDelta);
+ }
}
- private Map<String, Map<String, Map<String, ReplicaCount>>> computeReplicaCounts(List<Row> allRows,
- ComputedValueEvaluator computedValueEvaluator) {
- Map<String, Map<String, Map<String, ReplicaCount>>> collVsShardVsTagVsCount = new HashMap<>();
- for (Row row : allRows) {
- computedValueEvaluator.node = row.node;
- for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
- String collectionName = colls.getKey();
- if (!collection.isPass(collectionName)) continue;
- Map<String, Map<String, ReplicaCount>> collMap = collVsShardVsTagVsCount.computeIfAbsent(collectionName, s -> new HashMap<>());
- for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
- String shardName = shards.getKey();
- if (ANY.equals(shard.val)) shardName = ANY;
- if (!shard.isPass(shardName)) break;
- Map<String, ReplicaCount> tagVsCount = collMap.computeIfAbsent(shardName, s -> new HashMap<>());
- Object tagVal = row.getVal(tag.name);
- computedValueEvaluator.collName = collectionName;
- computedValueEvaluator.shardName = shardName;
- SealedClause sealedClause = getSealedClause(computedValueEvaluator);
- Condition t = sealedClause.getTag();
- if (t.varType.meta.isNodeSpecificVal()) {
- boolean pass = t.getOperand().match(t.val, tagVal) == TestStatus.PASS;
- tagVsCount.computeIfAbsent(row.node, s -> new ReplicaCount());
- if(pass) {
- tagVsCount.get(row.node).increment(shards.getValue());
- }
- } else {
- tagVsCount.computeIfAbsent(String.valueOf(t.getValue()), s -> new ReplicaCount());
- boolean pass = sealedClause.getTag().isPass(tagVal);
- if(!pass && !isReplicaZero()) continue;
- tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
- if (pass) {
- tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
- }
- }
+ void addViolatingReplicas(Condition tag,
+ ComputedValueEvaluator eval,
+ Violation.Ctx ctx, String tagName, Object tagVal,
+ Violation violation,
+ Policy.Session session) {
+ if (tag.varType.addViolatingReplicas(ctx)) return;
+ for (Row row : session.matrix) {
+ if (tagVal.equals(row.getVal(tagName))) {
+ row.forEachReplica(eval.collName, ri -> {
+ if (Policy.ANY.equals(eval.shardName)
+ || eval.shardName.equals(ri.getShard()))
+ violation.addReplica(new Violation.ReplicaInfoAndErr(ri).withDelta(tag.delta(row.getVal(tag.name))));
+ });
+ }
+ }
+
+ }
+
+ private void addReplicaCountsForNode(ComputedValueEvaluator computedValueEvaluator, ReplicaCount replicaCount, Row node) {
+ node.forEachReplica((String) collection.getValue(), ri -> {
+ if (Policy.ANY.equals(computedValueEvaluator.shardName)
+ || computedValueEvaluator.shardName.equals(ri.getShard()))
+ replicaCount.increment(ri);
+ });
+ }
+
+ List<Violation> testPerNode(Policy.Session session, double[] deviations) {
+ ComputedValueEvaluator eval = new ComputedValueEvaluator(session);
+ eval.collName = (String) collection.getValue();
+ Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, eval);
+ Set<String> shards = getShardNames(session, eval);
+ for (String s : shards) {
+ final ReplicaCount replicaCount = new ReplicaCount();
+ eval.shardName = s;
+ for (Row row : session.matrix) {
+ replicaCount.reset();
+ eval.node = row.node;
+ Condition tag = this.tag;
+ if (tag.computedType != null) {
+ tag = evaluateValue(tag, eval);
+ }
+ if (!tag.isPass(row)) continue;
+ addReplicaCountsForNode(eval, replicaCount, row);
+ SealedClause sealedClause = this.getSealedClause(eval);
+ if (!sealedClause.replica.isPass(replicaCount)) {
+ ReplicaCount replicaCountCopy = replicaCount.copy();
+ Violation violation = new Violation(sealedClause,
+ eval.collName,
+ eval.shardName,
+ eval.node,
+ replicaCountCopy,
+ sealedClause.getReplica().replicaCountDelta(replicaCountCopy),
+ eval.node);
+ ctx.resetAndAddViolation(row.node, replicaCountCopy, violation);
+ sealedClause.addViolatingReplicas(sealedClause.tag, eval, ctx, NODE, row.node, violation, session);
+ } else {
+ computeDeviation(deviations, replicaCount, sealedClause);
}
}
}
+ return ctx.allViolations;
+ }
+
+ private Set<String> getShardNames(Policy.Session session,
+ ComputedValueEvaluator eval) {
+ Set<String> shards = new HashSet<>();
+ if (isShardAbsent()) {
+ shards.add(Policy.ANY); //consider the entire collection is a single shard
+ } else {
+ for (Row row : session.matrix) {
+ row.forEachShard(eval.collName, (shard, r) -> {
+ if (this.shard.isPass(shard)) shards.add(shard); // add relevant shards
+ });
+ }
+ }
+ return shards;
+ }
+
+ boolean isShardAbsent() {
+ return Policy.ANY.equals(shard.val);
+ }
+
+ public List<Violation> test(Policy.Session session, double[] deviations) {
+ if (isPerCollectiontag()) {
+ return tag.varType == Type.NODE ||
+ (tag.varType.meta.isNodeSpecificVal() && replica.computedType == null) ?
+ testPerNode(session, deviations) :
+ testGroupNodes(session, deviations);
+ } else {
+ ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
+ Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator);
+ for (Row r : session.matrix) {
+ computedValueEvaluator.node = r.node;
+ SealedClause sealedClause = getSealedClause(computedValueEvaluator);
+ if (!sealedClause.getGlobalTag().isPass(r)) {
+ ctx.resetAndAddViolation(r.node, null, new Violation(sealedClause, null, null, r.node, r.getVal(sealedClause.globalTag.name),
+ sealedClause.globalTag.delta(r.getVal(globalTag.name)), r.node));
+ addViolatingReplicas(sealedClause.globalTag, computedValueEvaluator, ctx, Type.CORES.tagName, r.node, ctx.currentViolation, session);
+
+ }
+ }
+ return ctx.allViolations;
- if (this.getTag().op != LESS_THAN && this.getTag().varType == Type.NODE) {
- collVsShardVsTagVsCount.forEach((coll, shardVsNodeVsCount) ->
- shardVsNodeVsCount.forEach((shard, nodeVsCount) -> {
- for (Row row : allRows) {
- if (!nodeVsCount.containsKey(row.node)) {
- nodeVsCount.put(row.node, new ReplicaCount());
- }
- }
- }));
}
- return collVsShardVsTagVsCount;
}
+
public boolean isMatch(ReplicaInfo r, String collection, String shard) {
if (type != null && r.getType() != type) return false;
if (r.getCollection().equals(collection)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java
index 3a58804..5b60ef0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java
@@ -85,8 +85,12 @@ public class Condition implements MapWriter {
return false;
}
- public Double delta(Object val) {
+ public Double replicaCountDelta(Object val) {
if (val instanceof ReplicaCount) val = ((ReplicaCount) val).getVal(getClause().type);
+ return op.delta(this.val, val);
+ }
+
+ public Double delta(Object val) {
if (this.val instanceof String) {
if (op == LESS_THAN || op == GREATER_THAN) {
return op
@@ -96,15 +100,9 @@ public class Condition implements MapWriter {
return 0d;
}
} else {
- if (this == getClause().getReplica()) {
- Double delta = op.delta(this.val, val);
- return getClause().isReplicaZero() ? -1 * delta : delta;
- } else {
- return op
- .opposite(getClause().isReplicaZero() && this == getClause().getTag())
- .delta(this.val, val);
- }
-
+ return op
+ .opposite(getClause().isReplicaZero() && this == getClause().getTag())
+ .delta(this.val, val);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java
index 3344626..f8717b1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java
@@ -33,8 +33,7 @@ public class CoresVariable extends VariableBase {
return VariableBase.getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
}
- @Override
- public void addViolatingReplicas(Violation.Ctx ctx) {
+ public boolean addViolatingReplicas(Violation.Ctx ctx) {
for (Row row : ctx.allRows) {
if (row.node.equals(ctx.currentViolation.node)) {
row.forEachReplica(replicaInfo -> ctx.currentViolation
@@ -42,7 +41,7 @@ public class CoresVariable extends VariableBase {
.withDelta(ctx.currentViolation.replicaCountDelta)));
}
}
-
+ return true;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
index 600a708..600695a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
@@ -20,12 +20,16 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.Collectors;
+import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.util.Pair;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.suggestNegativeViolations;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORE_IDX;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.TOTALDISK;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -68,14 +72,12 @@ public class FreeDiskVariable extends VariableBase {
@Override
public void getSuggestions(Suggestion.Ctx ctx) {
if (ctx.violation == null) return;
- if (ctx.violation.replicaCountDelta < 0 && !ctx.violation.getViolatingReplicas().isEmpty()) {
-
- Comparator<Row> rowComparator = Comparator.comparing(r -> ((Double) r.getVal(ImplicitSnitch.DISK, 0d)));
+ if (ctx.violation.replicaCountDelta > 0) {
List<Row> matchingNodes = ctx.session.matrix.stream().filter(
row -> ctx.violation.getViolatingReplicas()
.stream()
.anyMatch(p -> row.node.equals(p.replicaInfo.getNode())))
- .sorted(rowComparator)
+ .sorted(Comparator.comparing(r -> ((Double) r.getVal(ImplicitSnitch.DISK, 0d))))
.collect(Collectors.toList());
@@ -94,16 +96,42 @@ public class FreeDiskVariable extends VariableBase {
if (currentDelta < 1) break;
if (replica.getVariables().get(CORE_IDX.tagName) == null) continue;
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
- .hint(Suggester.Hint.COLL_SHARD, new Pair<>(replica.getCollection(), replica.getShard()))
- .hint(Suggester.Hint.SRC_NODE, node.node)
+ .hint(Hint.COLL_SHARD, new Pair<>(replica.getCollection(), replica.getShard()))
+ .hint(Hint.SRC_NODE, node.node)
.forceOperation(true);
if (ctx.addSuggestion(suggester) == null) break;
currentDelta -= Clause.parseLong(CORE_IDX.tagName, replica.getVariable(CORE_IDX.tagName));
}
}
+ } else if (ctx.violation.replicaCountDelta < 0) {
+ suggestNegativeViolations(ctx, shards -> getSortedShards(ctx,shards));
}
}
+
+
+
+ private List<String> getSortedShards(Suggestion.Ctx ctx, Set<String> shardSet) {
+ return shardSet.stream()
+ .map(shard1 -> {
+ AtomicReference<Pair<String, Long>> result = new AtomicReference<>();
+ for (Row node : ctx.session.matrix) {
+ node.forEachShard(ctx.violation.coll, (s, ri) -> {
+ if (result.get() != null) return;
+ if (s.equals(shard1) && ri.size() > 0) {
+ Number sz = ((Number) ri.get(0).getVariable(CORE_IDX.tagName));
+ if (sz != null) result.set(new Pair<>(shard1, sz.longValue()));
+ }
+ });
+ }
+ return result.get() == null ? new Pair<>(shard1, 0L) : result.get();
+ })
+ .sorted(Comparator.comparingLong(Pair::second))
+ .map(Pair::first)
+ .collect(Collectors.toList());
+
+ }
+
//When a replica is added, freedisk should be incremented
@Override
public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> ops, boolean strictMode) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
index d4835b8..58b72bb 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
@@ -99,6 +99,7 @@ public enum Operand {
NOT_EQUAL("!", 2) {
@Override
public TestStatus match(Object ruleVal, Object testVal) {
+ if(testVal == null) return PASS;
return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
index 7e17df1..87fcf5a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
@@ -27,6 +27,16 @@ import org.apache.solr.common.util.Utils;
class ReplicaCount implements MapWriter {
long nrt, tlog, pull;
+ public ReplicaCount() {
+ nrt = tlog = pull = 0;
+ }
+
+ public ReplicaCount(long nrt, long tlog, long pull) {
+ this.nrt = nrt;
+ this.tlog = tlog;
+ this.pull = pull;
+ }
+
public long total() {
return nrt + tlog + pull;
}
@@ -55,19 +65,23 @@ class ReplicaCount implements MapWriter {
public void increment(List<ReplicaInfo> infos) {
if (infos == null) return;
for (ReplicaInfo info : infos) {
- switch (info.getType()) {
- case NRT:
- nrt++;
- break;
- case PULL:
- pull++;
- break;
- case TLOG:
- tlog++;
- break;
- default:
- nrt++;
- }
+ increment(info);
+ }
+ }
+
+ void increment(ReplicaInfo info) {
+ switch (info.getType()) {
+ case NRT:
+ nrt++;
+ break;
+ case PULL:
+ pull++;
+ break;
+ case TLOG:
+ tlog++;
+ break;
+ default:
+ nrt++;
}
}
@@ -75,4 +89,12 @@ class ReplicaCount implements MapWriter {
public String toString() {
return Utils.toJSONString(this);
}
+
+ public ReplicaCount copy() {
+ return new ReplicaCount(nrt, tlog, pull);
+ }
+
+ public void reset() {
+ nrt = tlog = pull = 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 88e9921..85d6f30 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -27,6 +27,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -71,6 +72,12 @@ public class Row implements MapWriter {
}
}
+ public void forEachShard(String collection, BiConsumer<String, List<ReplicaInfo>> consumer) {
+ collectionVsShardVsReplicas
+ .getOrDefault(collection, Collections.emptyMap())
+ .forEach(consumer);
+ }
+
public Row(String node, Cell[] cells, boolean anyValueMissing, Map<String,
Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, boolean isLive, Policy.Session session) {
this.session = session;
@@ -98,6 +105,7 @@ public class Row implements MapWriter {
}
Object getVal(String name) {
+ if (NODE.equals(name)) return this.node;
for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
return null;
}
@@ -128,11 +136,11 @@ public class Row implements MapWriter {
* values of certain attributes will be modified, in this node as well as other nodes. Please note that
* the state of the current session is kept intact while this operation is being performed
*
- * @param coll collection name
- * @param shard shard name
- * @param type replica type
+ * @param coll collection name
+ * @param shard shard name
+ * @param type replica type
* @param recursionCount the number of times we have recursed to add more replicas
- * @param strictMode whether suggester is operating in strict mode or not
+ * @param strictMode whether suggester is operating in strict mode or not
*/
Row addReplica(String coll, String shard, Replica.Type type, int recursionCount, boolean strictMode) {
if (recursionCount > 3) {
@@ -157,7 +165,7 @@ public class Row implements MapWriter {
if (op.isAdd) {
row = row.session.getNode(op.node).addReplica(op.coll, op.shard, op.type, recursionCount + 1, strictMode);
} else {
- row.session.getNode(op.node).removeReplica(op.coll, op.shard, op.type, recursionCount+1);
+ row.session.getNode(op.node).removeReplica(op.coll, op.shard, op.type, recursionCount + 1);
}
}
@@ -198,10 +206,12 @@ public class Row implements MapWriter {
if (idx == -1) return null;
return r.get(idx);
}
+
public Row removeReplica(String coll, String shard, Replica.Type type) {
- return removeReplica(coll,shard, type, 0);
+ return removeReplica(coll, shard, type, 0);
}
+
// this simulates removing a replica from a node
public Row removeReplica(String coll, String shard, Replica.Type type, int recursionCount) {
if (recursionCount > 3) {
@@ -240,10 +250,21 @@ public class Row implements MapWriter {
forEachReplica(collectionVsShardVsReplicas, consumer);
}
+ public void forEachReplica(String coll, Consumer<ReplicaInfo> consumer) {
+ collectionVsShardVsReplicas.getOrDefault(coll, Collections.emptyMap()).forEach((shard, replicaInfos) -> {
+ for (ReplicaInfo replicaInfo : replicaInfos) {
+ consumer.accept(replicaInfo);
+ }
+ });
+ }
+
public static void forEachReplica(Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, Consumer<ReplicaInfo> consumer) {
collectionVsShardVsReplicas.forEach((coll, shardVsReplicas) -> shardVsReplicas
.forEach((shard, replicaInfos) -> {
- for (ReplicaInfo r : replicaInfos) consumer.accept(r);
+ for (int i = 0; i < replicaInfos.size(); i++) {
+ ReplicaInfo r = replicaInfos.get(i);
+ consumer.accept(r);
+ }
}));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 3b18e02..1f711e5 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -18,7 +18,10 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.V2RequestSupport;
@@ -54,15 +57,51 @@ public class Suggestion {
}
}
+ static void suggestNegativeViolations(Suggestion.Ctx ctx, Function<Set<String>, List<String>> shardSorter) {
+ if (ctx.violation.coll == null) return;
+ Set<String> shardSet = new HashSet<>();
+ for (Row node : ctx.session.matrix)
+ node.forEachShard(ctx.violation.coll, (s, ri) -> {
+ if (Policy.ANY.equals(ctx.violation.shard) || s.equals(ctx.violation.shard)) shardSet.add(s);
+ });
+ //Now, sort shards based on their index size ascending
+ List<String> shards = shardSorter.apply(shardSet);
+ outer:
+ for (int i = 0; i < 5; i++) {
+ int totalSuggestions = 0;
+ for (String shard : shards) {
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .hint(Suggester.Hint.COLL_SHARD, new Pair<>(ctx.violation.coll, shard))
+ .forceOperation(true);
+ SolrRequest op = ctx.addSuggestion(suggester);
+ if (op == null) continue;
+ totalSuggestions++;
+ boolean violationStillExists = false;
+ for (Violation violation : suggester.session.getViolations()) {
+ if (violation.getClause().original == ctx.violation.getClause().original) {
+ violationStillExists = true;
+ break;
+ }
+ }
+ if (!violationStillExists) break outer;
+ }
+ if (totalSuggestions == 0) break;
+ }
+ }
- static void perNodeSuggestions(Ctx ctx) {
+
+ static void suggestPositiveViolations(Ctx ctx) {
if (ctx.violation == null) return;
+ Double currentDelta = ctx.violation.replicaCountDelta;
for (ReplicaInfoAndErr e : ctx.violation.getViolatingReplicas()) {
+ if (currentDelta <= 0) break;
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
.forceOperation(true)
.hint(Suggester.Hint.COLL_SHARD, new Pair<>(e.replicaInfo.getCollection(), e.replicaInfo.getShard()))
.hint(Suggester.Hint.SRC_NODE, e.replicaInfo.getNode());
- if (ctx.addSuggestion(suggester) == null) break;
+ if (ctx.addSuggestion(suggester) != null) {
+ currentDelta--;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
index 1ffb0a5..870483a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
@@ -31,8 +31,8 @@ import java.util.function.Consumer;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import static java.util.Collections.emptySet;
-import static java.util.Collections.unmodifiableSet;
import static java.util.Collections.unmodifiableMap;
+import static java.util.Collections.unmodifiableSet;
/**
* A Variable Type used in Autoscaling policy rules. Each variable type may have unique implementation
@@ -53,11 +53,8 @@ public interface Variable {
default void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode) {
}
- default void addViolatingReplicas(Violation.Ctx ctx) {
- for (Row row : ctx.allRows) {
- if (ctx.clause.tag.varType.meta.isNodeSpecificVal() && !row.node.equals(ctx.tagKey)) continue;
- Violation.collectViolatingReplicas(ctx, row);
- }
+ default boolean addViolatingReplicas(Violation.Ctx ctx) {
+ return false;
}
void getSuggestions(Suggestion.Ctx ctx);
@@ -85,7 +82,10 @@ public interface Variable {
* Type details of each variable in policies
*/
public enum Type implements Variable {
- @Meta(name = "withCollection", type = String.class, isNodeSpecificVal = true, implementation = WithCollectionVariable.class)
+ @Meta(name = "withCollection",
+ type = String.class,
+ isNodeSpecificVal = true,
+ implementation = WithCollectionVariable.class)
WITH_COLLECTION(),
@Meta(name = "collection",
@@ -285,8 +285,8 @@ public interface Variable {
}
@Override
- public void addViolatingReplicas(Violation.Ctx ctx) {
- impl.addViolatingReplicas(ctx);
+ public boolean addViolatingReplicas(Violation.Ctx ctx) {
+ return impl.addViolatingReplicas(ctx);
}
public Operand getOperand(Operand expected, Object val, ComputedType computedType) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
index 82a5ce6..aaa874d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
@@ -18,11 +18,14 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
+import java.util.ArrayList;
+
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.util.StrUtils;
import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.parseString;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.perNodeSuggestions;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.suggestNegativeViolations;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.suggestPositiveViolations;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
public class VariableBase implements Variable {
@@ -34,7 +37,12 @@ public class VariableBase implements Variable {
@Override
public void getSuggestions(Suggestion.Ctx ctx) {
- perNodeSuggestions(ctx);
+ if (ctx.violation == null) return;
+ if (ctx.violation.replicaCountDelta > 0) {
+ suggestPositiveViolations(ctx);
+ } else {
+ suggestNegativeViolations(ctx, ArrayList::new);
+ }
}
static Object getOperandAdjustedValue(Object val, Object original) {
@@ -177,7 +185,7 @@ public class VariableBase implements Variable {
@Override
public void getSuggestions(Suggestion.Ctx ctx) {
- perNodeSuggestions(ctx);
+ suggestPositiveViolations(ctx);
}
}
@@ -188,7 +196,7 @@ public class VariableBase implements Variable {
@Override
public void getSuggestions(Suggestion.Ctx ctx) {
- perNodeSuggestions(ctx);
+ suggestPositiveViolations(ctx);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 53f7924..e0d2048 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -49,29 +49,6 @@ public class Violation implements MapWriter {
hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
}
- static void collectViolatingReplicas(Ctx ctx, Row row) {
- if (ctx.clause.tag.varType.meta.isNodeSpecificVal()) {
- row.forEachReplica(replica -> {
- if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard())) {
- ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica)
- .withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
- }
- });
- } else {
- row.forEachReplica(replica -> {
- if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
- if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
- if(!ctx.currentViolation.getClause().matchShard(replica.getShard(), ctx.currentViolation.shard)) return;
- if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
- return;
- ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
- });
-
- }
-
-
- }
-
public Violation addReplica(ReplicaInfoAndErr r) {
replicaInfoAndErrs.add(r);
return this;
@@ -159,9 +136,9 @@ public class Violation implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.putIfNotNull("collection", coll);
- ew.putIfNotNull("shard", shard);
+ if (!Policy.ANY.equals(shard)) ew.putIfNotNull("shard", shard);
ew.putIfNotNull("node", node);
- ew.putStringIfNotNull("tagKey", tagKey);
+ ew.putIfNotNull("tagKey", tagKey);
ew.putIfNotNull("violation", (MapWriter) ew1 -> {
if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal);
else ew1.put(clause.tag.name, String.valueOf(actualVal));
@@ -179,7 +156,7 @@ public class Violation implements MapWriter {
static class Ctx {
final Function<Condition, Object> evaluator;
- String tagKey;
+ Object tagKey;
Clause clause;
ReplicaCount count;
Violation currentViolation;
@@ -192,7 +169,7 @@ public class Violation implements MapWriter {
this.evaluator = evaluator;
}
- public Ctx reset(String tagKey, ReplicaCount count, Violation currentViolation) {
+ public Ctx resetAndAddViolation(Object tagKey, ReplicaCount count, Violation currentViolation) {
this.tagKey = tagKey;
this.count = count;
this.currentViolation = currentViolation;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
index b295aee..db50726 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
@@ -82,7 +82,7 @@ public class WithCollectionVariable extends VariableBase {
return Integer.compare(v1.getViolatingReplicas().size(), v2.getViolatingReplicas().size());
}
- public void addViolatingReplicas(Violation.Ctx ctx) {
+ public boolean addViolatingReplicas(Violation.Ctx ctx) {
String node = ctx.currentViolation.node;
for (Row row : ctx.allRows) {
if (node.equals(row.node)) {
@@ -103,6 +103,7 @@ public class WithCollectionVariable extends VariableBase {
}
}
}
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index b75e1ba..a48141e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -1377,9 +1377,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}");
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3, sysprop.fs: unknown }," +
"node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1, sysprop.fs: unknown}" +
"}");
Policy policy = new Policy(policies);
Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, clusterState))
@@ -1573,9 +1573,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}");
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3, sysprop.fs: unknown}," +
"node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1, sysprop.fs: unknown}" +
"}");
Policy policy = new Policy(policies);
Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, clusterState))
@@ -1889,10 +1889,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
" {'minimize':'cores', 'precision':3}," +
" {'maximize':'freedisk','precision':100}]}";
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3, sysprop.fs: slowdisk}," +
"node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1, sysprop.fs: slowdisk}" +
"}");
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
SolrCloudManager cloudManager = getSolrCloudManager(nodeValues, clusterState);
@@ -2526,7 +2526,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
- public void testFreeDiskSuggestions() {
+ public void testFreeDiskSuggestions() throws IOException {
String dataproviderdata = "{" +
" liveNodes:[node1,node2]," +
" replicaInfo : {" +
@@ -2549,7 +2549,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
assertEquals(1, violations.size());
assertEquals(4, violations.get(0).getViolatingReplicas().size());
- assertEquals(-4, violations.get(0).replicaCountDelta, 0.1);
+ assertEquals(4, violations.get(0).replicaCountDelta, 0.1);
for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
assertEquals(500d, r.delta, 0.1);
@@ -2579,11 +2579,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(1, violations.size());
assertEquals(-4, violations.get(0).replicaCountDelta, 0.1);
assertEquals(1, violations.size());
- assertEquals(4, violations.get(0).getViolatingReplicas().size());
- for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
- assertEquals(500d, r.delta, 0.1);
+ assertEquals(0, violations.get(0).getViolatingReplicas().size());
- }
l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(3, l.size());
m = l.get(0).toMap(new LinkedHashMap<>());
@@ -3653,7 +3650,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) Utils.fromJSONString(dataproviderdata))).getViolations();
assertEquals(1, violations.size());
- assertEquals(4, violations.get(0).getViolatingReplicas().size());
+ assertEquals(-4d, violations.get(0).replicaCountDelta, 0.01);
for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
assertEquals(10.0d, r.delta.doubleValue(), 0.1);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6cdcf09f/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index d6a5d51..5365e28 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -31,6 +31,7 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import com.google.common.collect.ImmutableSet;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
@@ -57,13 +58,13 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
" 'shard1': {" +
" 'range': '80000000-ffffffff'," +
" 'replicas': {" +
- " 'r1': {" +
+ " 'r1': {" +//east
" 'core': 'r1'," +
" 'base_url': 'http://10.0.0.4:8983/solr'," +
" 'node_name': 'node1'," +
" 'state': 'active'" +
" }," +
- " 'r2': {" +
+ " 'r2': {" +//west
" 'core': 'r2'," +
" 'base_url': 'http://10.0.0.4:7574/solr'," +
" 'node_name': 'node2'," +
@@ -74,25 +75,25 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
" 'shard2': {" +
" 'range': '0-7fffffff'," +
" 'replicas': {" +
- " 'r3': {" +
+ " 'r3': {" +//east
" 'core': 'r3'," +
" 'base_url': 'http://10.0.0.4:8983/solr'," +
" 'node_name': 'node1'," +
" 'state': 'active'" +
" }," +
- " 'r4': {" +
+ " 'r4': {" +//west
" 'core': 'r4'," +
" 'base_url': 'http://10.0.0.4:8987/solr'," +
" 'node_name': 'node4'," +
" 'state': 'active'" +
" }," +
- " 'r6': {" +
+ " 'r6': {" +//east
" 'core': 'r6'," +
" 'base_url': 'http://10.0.0.4:8989/solr'," +
" 'node_name': 'node3'," +
" 'state': 'active'" +
" }," +
- " 'r5': {" +
+ " 'r5': {" +//east
" 'core': 'r5'," +
" 'base_url': 'http://10.0.0.4:8983/solr'," +
" 'node_name': 'node1'," +
@@ -126,7 +127,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
Policy.Session session = policy.createSession(createCloudManager(state, metaData));
List<Violation> violations = session.getViolations();
assertEquals(1, violations.size());
- assertEquals(4, violations.get(0).getViolatingReplicas().size());
+ assertEquals(3, violations.get(0).getViolatingReplicas().size());
assertEquals(1.0, violations.get(0).replicaCountDelta, 0.01);
for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
assertEquals("shard2", r.replicaInfo.getShard());
@@ -139,7 +140,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
session = policy.createSession(createCloudManager(state, metaData));
violations = session.getViolations();
assertEquals(1, violations.size());
- assertEquals(4, violations.get(0).getViolatingReplicas().size());
+ assertEquals(3, violations.get(0).getViolatingReplicas().size());
assertEquals(1.0, violations.get(0).replicaCountDelta, 0.01);
for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
assertEquals("shard2", r.replicaInfo.getShard());
@@ -526,6 +527,182 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
};
}
+ public void testSysPropSuggestions() {
+ String diagnostics = "{" +
+ " 'diagnostics': {" +
+ " 'sortedNodes': [" +
+ " {" +
+ " 'node': '127.0.0.1:63191_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'sysprop.zone': 'east'," +
+ " 'freedisk': 1727.1459312438965," +
+ " 'heapUsage': 24.97510064011647," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'zonesTest': {" +
+ " 'shard1': [" +
+ " {" +
+ " 'core_node5': {" +
+ " 'core': 'zonesTest_shard1_replica_n2'," +
+ " 'leader': 'true'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'zonesTest'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node7': {" +
+ " 'core': 'zonesTest_shard1_replica_n4'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'zonesTest'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node12': {" +
+ " 'core': 'zonesTest_shard1_replica_n10'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'zonesTest'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63192_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'sysprop.zone': 'east'," +
+ " 'freedisk': 1727.1459312438965," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'zonesTest': {" +
+ " 'shard2': [" +
+ " {" +
+ " 'core_node3': {" +
+ " 'core': 'zonesTest_shard1_replica_n1'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard2'," +
+ " 'collection': 'zonesTest'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node9': {" +
+ " 'core': 'zonesTest_shard1_replica_n6'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard2'," +
+ " 'collection': 'zonesTest'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node11': {" +
+ " 'core': 'zonesTest_shard1_replica_n8'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard2'," +
+ " 'collection': 'zonesTest'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63219_solr'," +
+ " 'isLive': true," +
+ " 'cores': 0.0," +
+ " 'sysprop.zone': 'west'," +
+ " 'freedisk': 1768.6174201965332," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {}" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63229_solr'," +
+ " 'isLive': true," +
+ " 'cores': 0.0," +
+ " 'sysprop.zone': 'west'," +
+ " 'freedisk': 1768.6174201965332," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {}" +
+ " }" +
+ " ]," +
+ " 'liveNodes': [" +
+ " '127.0.0.1:63191_solr'," +
+ " '127.0.0.1:63192_solr'," +
+ " '127.0.0.1:63219_solr'," +
+ " '127.0.0.1:63229_solr'" +
+ " ]," +
+ " 'config': {" +
+ " 'cluster-preferences': [" +
+ " {'minimize': 'cores', 'precision': 1}," +
+ " {'maximize': 'freedisk', 'precision': 100}," +
+ " {'minimize': 'sysLoadAvg', 'precision': 10}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " {'replica': '<3', 'shard': '#EACH', 'sysprop.zone': [east, west]}" +
+ " ]" +
+ " }" +
+ " }" +
+ "}";
+
+ Map<String, Object> m = (Map<String, Object>) Utils.fromJSONString(diagnostics);
+
+ Map<String, Object> conf = (Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config");
+ Policy policy = new Policy(conf);
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+ Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
+ List<Violation> violations = session.getViolations();
+ for (Violation violation : violations) {
+ assertEquals(1.0d, violation.replicaCountDelta.doubleValue(), 0.0001);
+ }
+ assertEquals(2, violations.size());
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig(conf), cloudManagerFromDiagnostics);
+ assertEquals(2, suggestions.size());
+ for (Suggester.SuggestionInfo suggestion : suggestions) {
+ assertTrue(ImmutableSet.of("127.0.0.1:63219_solr", "127.0.0.1:63229_solr").contains(
+ Utils.getObjectByPath(suggestion, true, "operation/command/move-replica/targetNode")));
+
+ }
+ }
}