You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2018/08/08 16:34:09 UTC
lucene-solr:branch_7x: SOLR-12618: AutoScalingHandlerTest failing in
jenkins
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 6e720b909 -> d087f9f1e
SOLR-12618: AutoScalingHandlerTest failing in jenkins
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d087f9f1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d087f9f1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d087f9f1
Branch: refs/heads/branch_7x
Commit: d087f9f1ef065b449c93514e21c859e6dd0f7ad7
Parents: 6e720b9
Author: Noble Paul <no...@apache.org>
Authored: Thu Aug 9 01:24:03 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Thu Aug 9 02:33:50 2018 +1000
----------------------------------------------------------------------
.../autoscaling/AutoScalingHandlerTest.java | 43 ++--
.../cloud/autoscaling/AddReplicaSuggester.java | 10 +-
.../client/solrj/cloud/autoscaling/Clause.java | 18 +-
.../cloud/autoscaling/MoveReplicaSuggester.java | 3 +-
.../solrj/cloud/autoscaling/NodeVariable.java | 5 +-
.../client/solrj/cloud/autoscaling/Policy.java | 2 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 38 ++-
.../solrj/cloud/autoscaling/RangeVal.java | 10 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 6 +-
.../solrj/cloud/autoscaling/ReplicaInfo.java | 17 +-
.../solrj/cloud/autoscaling/Suggester.java | 14 +-
.../solrj/cloud/autoscaling/VariableBase.java | 1 +
.../solrj/cloud/autoscaling/Violation.java | 10 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 2 +-
.../solr/common/ConditionalMapWriter.java | 75 ++++++
.../java/org/apache/solr/common/MapWriter.java | 6 +
.../solrj/cloud/autoscaling/TestPolicy.java | 16 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 240 +++++++++++++++++++
.../apache/solr/cloud/SolrCloudTestCase.java | 19 ++
19 files changed, 462 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 5b8ec21..b9e8e66 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -675,7 +676,6 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
public void testReadApi() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
// first trigger
@@ -693,9 +693,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
String setClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
+ " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}]" +
"}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
response = solrClient.request(req);
@@ -713,17 +711,9 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertEquals(response.get("result").toString(), "success");
String setPolicyCommand = "{'set-policy': {" +
- " 'xyz':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]," +
- " 'policy1':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]," +
- " 'policy2':[" +
- " {'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]" +
- "}}";
+ " 'xyz':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
+ " 'policy1':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
+ " 'policy2':[{'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}]}}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setPolicyCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -747,7 +737,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
List<Map> clusterPolicy = (List<Map>) response.get("cluster-policy");
assertNotNull(clusterPolicy);
- assertEquals(3, clusterPolicy.size());
+ assertEquals(2, clusterPolicy.size());
Map policies = (Map) response.get("policies");
assertNotNull(policies);
@@ -766,11 +756,11 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
for (int i = 0; i < 2; i++) {
Map node = (Map) sortedNodes.get(i);
assertNotNull(node);
- assertEquals(6, node.size());
assertNotNull(node.get("node"));
assertNotNull(node.get("cores"));
assertEquals(0d, node.get("cores"));
assertNotNull(node.get("freedisk"));
+ assertNotNull(node.get("replicas"));
assertTrue(node.get("freedisk") instanceof Double);
assertNotNull(node.get("sysLoadAvg"));
assertTrue(node.get("sysLoadAvg") instanceof Double);
@@ -800,8 +790,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
String tempClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
+ " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}"+
" ]" +
"}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, tempClusterPolicyCommand);
@@ -831,12 +820,26 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
for (Map<String, Object> violation : violations) {
assertEquals("readApiTestViolations", violation.get("collection"));
assertEquals("shard1", violation.get("shard"));
- assertEquals(2d, getObjectByPath(violation, true, "violation/delta"));
+ assertEquals(1.0d, getObjectByPath(violation, true, "violation/delta"));
assertEquals(3l, getObjectByPath(violation, true, "violation/replica/NRT"));
assertNotNull(violation.get("clause"));
}
+ log.info("Before starting new jetty ,{}", cluster.getJettySolrRunners()
+ .stream()
+ .map(jettySolrRunner -> jettySolrRunner.getNodeName()).collect(Collectors.toList()));
JettySolrRunner runner1 = cluster.startJettySolrRunner();
cluster.waitForAllNodes(30);
+ log.info("started new jetty {}", runner1.getNodeName());
+
+ response = waitForResponse(namedList -> {
+ List l = (List) Utils.getObjectByPath(namedList, false, "diagnostics/liveNodes");
+ if (l != null && l.contains(runner1.getNodeName())) return true;
+ return false;
+ },
+ createAutoScalingRequest(SolrRequest.METHOD.GET, "/diagnostics", null),
+ 200,
+ 20,
+ runner1.getNodeName() + " could not come up ");
req = createAutoScalingRequest(SolrRequest.METHOD.GET, "/suggestions", null);
response = solrClient.request(req);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
index 2a31e71..a498932 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@ -47,16 +47,19 @@ class AddReplicaSuggester extends Suggester {
//iterate through nodes and identify the least loaded
List<Violation> leastSeriousViolation = null;
Row bestNode = null;
+ double[] bestDeviation = null;
for (int i = getMatrix().size() - 1; i >= 0; i--) {
Row row = getMatrix().get(i);
if (!isNodeSuitableForReplicaAddition(row)) continue;
Row tmpRow = row.addReplica(shard.first(), shard.second(), type, strict);
- List<Violation> errs = testChangedMatrix(strict, tmpRow.session);
-
+ double[] deviation = new double[1];
+ List<Violation> errs = testChangedMatrix(strict, tmpRow.session, deviation);
if (!containsNewErrors(errs)) {
- if (isLessSerious(errs, leastSeriousViolation)) {
+ if ((errs.isEmpty() && isLessDeviant(bestDeviation, deviation)) ||//there are no violations but this is deviating less
+ isLessSerious(errs, leastSeriousViolation)) {//there are errors , but this has less serious violation
leastSeriousViolation = errs;
bestNode = tmpRow;
+ bestDeviation = deviation;
}
}
}
@@ -73,6 +76,7 @@ class AddReplicaSuggester extends Suggester {
return null;
}
+
@Override
public CollectionParams.CollectionAction getAction() {
return ADDREPLICA;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index cd9212b..e325d10 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -42,6 +42,7 @@ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.RANGE_EQUAL;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
@@ -215,7 +216,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
if (v == 0) {// higher the number of replicas , harder to satisfy
- v = Preference.compareWithTolerance((Double) this.replica.val, (Double) that.replica.val, 1);
+ Double thisVal = this.replica.val instanceof RangeVal ? ((RangeVal) this.replica.val).max.doubleValue() : (Double) this.replica.val;
+ Double thatVal = that.replica.val instanceof RangeVal ? ((RangeVal) that.replica.val).max.doubleValue() : (Double) that.replica.val;
+ v = Preference.compareWithTolerance(thisVal, thatVal, 1);
v = this.replica.op == LESS_THAN ? v : v * -1;
}
if (v == 0) v = compareTypes(this.type, that.type);
@@ -240,7 +243,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
//replica value is zero
boolean isReplicaZero() {
- return replica != null && replica.getOperand() == Operand.EQUAL &&
+ return replica != null && replica.getOperand() == Operand.EQUAL && replica.val instanceof Double &&
Preference.compareWithTolerance(0d, (Double) replica.val, 1) == 0;
}
@@ -353,7 +356,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
return operand;
}
- public List<Violation> test(Policy.Session session) {
+ public List<Violation> test(Policy.Session session, double[] deviations) {
ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator);
if (isPerCollectiontag()) {
@@ -377,6 +380,13 @@ public class Clause implements MapWriter, Comparable<Clause> {
sealedClause.getReplica().delta(replicas),
tag.varType.meta.isNodeSpecificVal() ? null : counts.getKey());
tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
+ } else {
+ if (deviations != null && sealedClause.replica.op == RANGE_EQUAL) {
+ Number actualCount = replicas.getVal(type);
+ Double realDelta = ((RangeVal) sealedClause.replica.val).realDelta(actualCount.doubleValue());
+ realDelta = this.isReplicaZero() ? -1 * realDelta : realDelta;
+ deviations[0] += Math.abs(realDelta);
+ }
}
}
}
@@ -422,7 +432,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
tagVsCount.get(row.node).increment(shards.getValue());
}
} else {
+ tagVsCount.computeIfAbsent(String.valueOf(t.getValue()), s -> new ReplicaCount());
boolean pass = sealedClause.getTag().isPass(tagVal);
+ if(!pass && !isReplicaZero()) continue;
tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
if (pass) {
tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
index fffaee2..9e778c0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@ -62,7 +62,8 @@ public class MoveReplicaSuggester extends Suggester {
if (!isNodeSuitableForReplicaAddition(targetRow)) continue;
targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first
Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node
- List<Violation> errs = testChangedMatrix(strict, srcRowModified.session);
+ double[] deviation = new double[1];
+ List<Violation> errs = testChangedMatrix(strict, srcRowModified.session, deviation);
srcRowModified.session.applyRules(); // now resort the nodes with the new values
Policy.Session tmpSession = srcRowModified.session;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
index 3292800..48dcd5e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
@@ -33,13 +33,12 @@ public class NodeVariable extends VariableBase {
if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .forceOperation(true)
.hint(Suggester.Hint.SRC_NODE, ctx.violation.node)
.hint(ctx.violation.shard.equals(ANY) ? Suggester.Hint.COLL : Suggester.Hint.COLL_SHARD,
ctx.violation.shard.equals(ANY) ? ctx.violation.coll : new Pair<>(ctx.violation.coll, ctx.violation.shard));
- ctx.addSuggestion(suggester);
+ if(ctx.addSuggestion(suggester) == null) break;
}
}
-
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index b398caf..8b0a956 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -572,7 +572,7 @@ public class Policy implements MapWriter {
setApproxValuesAndSortNodes(clusterPreferences, matrix);
for (Clause clause : expandedClauses) {
- List<Violation> errs = clause.test(this);
+ List<Violation> errs = clause.test(this, null);
violations.addAll(errs);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index e855f8a..ab48163 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -28,12 +28,14 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
@@ -183,30 +185,20 @@ public class PolicyHelper {
public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) {
Policy.Session session = policy.createSession(cloudManager);
List<Row> sorted = session.getSortedNodes();
- List<Violation> violations = session.getViolations();
-
- List<Preference> clusterPreferences = policy.getClusterPreferences();
-
- List<Map<String, Object>> sortedNodes = new ArrayList<>(sorted.size());
- for (Row row : sorted) {
- Map<String, Object> map = Utils.makeMap("node", row.node);
- map.put("isLive", row.isLive);
- for (Cell cell : row.getCells()) {
- for (Preference clusterPreference : clusterPreferences) {
- Policy.SortParam name = clusterPreference.getName();
- if (cell.getName().equalsIgnoreCase(name.name())) {
- map.put(name.name(), cell.getValue());
- break;
- }
- }
+ return ew -> ew.put("sortedNodes", (IteratorWriter) iw -> {
+ for (Row row : sorted) {
+ iw.add((MapWriter) ew1 -> {
+ ew1.put("node", row.node).
+ put("isLive", row.isLive);
+ for (Cell cell : row.getCells())
+ ew1.put(cell.name, cell.val,
+ (Predicate) o -> o != null && (!(o instanceof Map) || !((Map) o).isEmpty()));
+ ew1.put("replicas", row.collectionVsShardVsReplicas);
+ });
}
- sortedNodes.add(map);
- }
-
- return ew -> {
- ew.put("sortedNodes", sortedNodes);
- ew.put("violations", violations);
- };
+ }).put("liveNodes", cloudManager.getClusterStateProvider().getLiveNodes())
+ .put("violations", session.getViolations())
+ .put("config", session.getPolicy());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
index 16cfe6a..b9f6b8d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
@@ -35,8 +35,12 @@ class RangeVal implements MapWriter {
Double.compare(testVal.doubleValue(), max.doubleValue()) <= 0;
}
+ public Double realDelta(double v) {
+ if (actual != null) return v - actual.doubleValue();
+ else return delta(v);
+ }
+
public Double delta(double v) {
-// if (actual != null) return v - actual.doubleValue();
if (v >= max.doubleValue()) return v - max.doubleValue();
if (v <= min.doubleValue()) return v - min.doubleValue();
return 0d;
@@ -49,6 +53,8 @@ class RangeVal implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put("min", min).put("max", max).putIfNotNull("actual", actual);
+ ew.put("min", min)
+ .put("max", max)
+ .putIfNotNull("actual", actual);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
index 588508f..7e17df1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
@@ -33,9 +33,9 @@ class ReplicaCount implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put(Replica.Type.NRT.name(), nrt);
- ew.put(Replica.Type.PULL.name(), pull);
- ew.put(Replica.Type.TLOG.name(), tlog);
+ if (nrt > 0) ew.put(Replica.Type.NRT.name(), nrt);
+ if (pull > 0) ew.put(Replica.Type.PULL.name(), pull);
+ if (tlog > 0) ew.put(Replica.Type.TLOG.name(), tlog);
ew.put("count", total());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
index edc7269..25f12bd 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
@@ -53,7 +53,7 @@ public class ReplicaInfo implements MapWriter {
}
public ReplicaInfo(String name, String core, String coll, String shard, Replica.Type type, String node, Map<String, Object> vals) {
- if(vals==null) vals = Collections.emptyMap();
+ if (vals == null) vals = Collections.emptyMap();
this.name = name;
if (vals != null) {
this.variables.putAll(vals);
@@ -66,6 +66,21 @@ public class ReplicaInfo implements MapWriter {
this.node = node;
}
+ ReplicaInfo(Map<String, Object> map) {
+ this.name = map.keySet().iterator().next();
+ Map details = (Map) map.get(name);
+ details = Utils.getDeepCopy(details, 4);
+ this.collection = (String) details.remove("collection");
+ this.shard = (String) details.remove("shard");
+ this.core = (String) details.remove("core");
+ this.node = (String) details.remove("node_name");
+ this.isLeader = Boolean.parseBoolean((String) details.getOrDefault("leader", "false"));
+ details.remove("leader");
+ type = Replica.Type.valueOf((String) details.getOrDefault("type", "NRT"));
+ details.remove("type");
+ this.variables.putAll(details);
+ }
+
public Object clone() {
return new ReplicaInfo(name, core, collection, shard, type, node, variables);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index 9b9b60e..db0aab4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -35,6 +35,7 @@ import java.util.function.Predicate;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.common.ConditionalMapWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
@@ -70,6 +71,10 @@ public abstract class Suggester implements MapWriter {
this.session = session.copy();
}
+ boolean isLessDeviant(double[] previousBest, double[] newDeviation) {
+ if (previousBest == null) return true;
+ return newDeviation[0] < previousBest[0];
+ }
public Suggester hint(Hint hint, Object value) {
hint.validator.accept(value);
if (hint.multiValued) {
@@ -203,7 +208,9 @@ public abstract class Suggester implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put("type", violation == null ? "improvement" : "violation");
- ew.putIfNotNull("violation", violation);
+ ew.putIfNotNull("violation",
+ new ConditionalMapWriter(violation,
+ (k, v) -> !"violatingReplicas".equals(k)));
ew.put("operation", operation);
}
@@ -237,6 +244,7 @@ public abstract class Suggester implements MapWriter {
boolean containsNewErrors(List<Violation> violations) {
boolean isTxOpen = session.transaction != null && session.transaction.isOpen();
+ if (violations.size() > originalViolations.size()) return true;
for (Violation v : violations) {
//the computed value can change over time. So it's better to evaluate it in the end
if (isTxOpen && v.getClause().hasComputedValue) continue;
@@ -272,12 +280,12 @@ public abstract class Suggester implements MapWriter {
}
}
- List<Violation> testChangedMatrix(boolean strict, Policy.Session session) {
+ List<Violation> testChangedMatrix(boolean strict, Policy.Session session, double[] deviation) {
Policy.setApproxValuesAndSortNodes(session.getPolicy().clusterPreferences, session.matrix);
List<Violation> errors = new ArrayList<>();
for (Clause clause : session.expandedClauses) {
if (strict || clause.strict) {
- List<Violation> errs = clause.test(session);
+ List<Violation> errs = clause.test(session, deviation);
if (!errs.isEmpty()) {
errors.addAll(errs);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
index 8b3f1e1..4387eb1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
@@ -107,6 +107,7 @@ public class VariableBase implements Variable {
val = condition.op.readRuleValue(condition);
if (val != condition.val) return val;
}
+ if (!isRuleVal && "".equals(val) && this.varType.type != String.class) val = -1;
if (name == null) name = this.varType.tagName;
if (varType.type == Double.class) {
Double num = Clause.parseDouble(name, val);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 39b3c0b..53f7924 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Objects;
import java.util.function.Function;
+import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.Utils;
@@ -160,13 +161,20 @@ public class Violation implements MapWriter {
ew.putIfNotNull("collection", coll);
ew.putIfNotNull("shard", shard);
ew.putIfNotNull("node", node);
- ew.putIfNotNull("tagKey", String.valueOf(tagKey));
+ ew.putStringIfNotNull("tagKey", tagKey);
ew.putIfNotNull("violation", (MapWriter) ew1 -> {
if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal);
else ew1.put(clause.tag.name, String.valueOf(actualVal));
ew1.putIfNotNull("delta", replicaCountDelta);
});
ew.put("clause", getClause());
+ if (!replicaInfoAndErrs.isEmpty()) {
+ ew.put("violatingReplicas", (IteratorWriter) iw -> {
+ for (ReplicaInfoAndErr replicaInfoAndErr : replicaInfoAndErrs) {
+ iw.add(replicaInfoAndErr.replicaInfo);
+ }
+ });
+ }
}
static class Ctx {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 83fb25a..d148e21 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -94,7 +94,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return solrClient.getClusterStateProvider();
}
- private void readReplicaDetails() throws IOException {
+ protected void readReplicaDetails() throws IOException {
ClusterStateProvider clusterStateProvider = getClusterStateProvider();
ClusterState clusterState = clusterStateProvider.getClusterState();
if (clusterState == null) { // zkStateReader still initializing
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
new file mode 100644
index 0000000..9a1106f
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.IOException;
+import java.util.function.BiPredicate;
+
+public class ConditionalMapWriter implements MapWriter {
+ private final MapWriter delegate;
+ private final BiPredicate<String, Object> predicate;
+
+ public ConditionalMapWriter(MapWriter delegate, BiPredicate<String, Object> predicate) {
+ this.delegate = delegate;
+ this.predicate = predicate;
+ }
+
+ private class EntryWriterWrapper implements EntryWriter {
+ private final EntryWriter delegate;
+
+ EntryWriterWrapper(EntryWriter delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public EntryWriter put(String k, Object v) throws IOException {
+ if (predicate.test(k, v)) delegate.put(k, v);
+ return this;
+ }
+
+ @Override
+ public EntryWriter put(String k, int v) throws IOException {
+ return put(k, Integer.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, long v) throws IOException {
+ return put(k, Long.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, float v) throws IOException {
+ return put(k, Float.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, double v) throws IOException {
+ return put(k, Double.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, boolean v) throws IOException {
+ return put(k, Boolean.valueOf(v));
+ }
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ delegate.writeMap(new EntryWriterWrapper(ew));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index fc57e3d..56f3b8d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Predicate;
import org.apache.solr.common.util.Utils;
@@ -102,6 +103,11 @@ public interface MapWriter extends MapSerializable {
return this;
}
+ default EntryWriter put(String k, Object v, Predicate<Object> p) throws IOException {
+ if (p.test(v)) put(k, v);
+ return this;
+ }
+
default EntryWriter putIfNotNull(String k, Object v) throws IOException {
if(v != null) put(k,v);
return this;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 3bd58aa..19db8fc 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -2467,8 +2467,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
" }}}";
String autoScalingjson = " { cluster-policy:[" +
- " { replica :'<34%', shard: '#EACH', sysprop.az : east}," +
- " { replica :'<67%', shard: '#EACH', sysprop.az : west}" +
+ " { replica :'33%', shard: '#EACH', sysprop.az : east}," +
+ " { replica :'67%', shard: '#EACH', sysprop.az : west}" +
" ]," +
" cluster-preferences :[{ minimize : cores }]}";
@@ -2480,6 +2480,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<String> nodes = new ArrayList<>();
+ int westCount = 0, eastCount = 0;
for (int i = 0; i < 12; i++) {
SolrRequest suggestion = txn.getCurrentSession()
.getSuggester(ADDREPLICA)
@@ -2488,9 +2489,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(suggestion);
String node = suggestion.getParams().get("node");
nodes.add(node);
+ if ("10.0.0.6:8983_solr".equals(node)) eastCount++;
+ if ("10.0.0.6:7574_solr".equals(node)) westCount++;
if (i % 3 == 1) assertEquals("10.0.0.6:8983_solr", node);
else assertEquals("10.0.0.6:7574_solr", node);
}
+ assertEquals(8, westCount);
+ assertEquals(4, eastCount);
List<Violation> violations = txn.close();
assertTrue(violations.isEmpty());
@@ -2655,7 +2660,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}";
AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
- assertEquals(2, violations.size());
+ assertEquals("expected 2 violations", 2, violations.size());
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(2, suggestions.size());
for (Suggester.SuggestionInfo suggestion : suggestions) {
@@ -3562,11 +3567,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
Object root = Utils.fromJSONString(writer.toString());
assertEquals(2l,
Utils.getObjectByPath(root, true, "violations[0]/violation/replica/NRT"));
- assertEquals(0l,
- Utils.getObjectByPath(root, true, "violations[0]/violation/replica/PULL"));
- assertEquals(0l,
- Utils.getObjectByPath(root, true, "violations[0]/violation/replica/TLOG"));
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index c1d69b9..d6a5d51 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -288,4 +289,243 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
};
}
+ public void testAutoScalingHandlerFailure() {
+ String diagnostics = "{" +
+ " 'diagnostics': {" +
+ " 'sortedNodes': [" +
+ " {" +
+ " 'node': '127.0.0.1:63191_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.97510064011647," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'readApiTestViolations': {" +
+ " 'shard1': [" +
+ " {" +
+ " 'core_node5': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n2'," +
+ " 'leader': 'true'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node7': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n4'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node12': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n10'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63192_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'readApiTestViolations': {" +
+ " 'shard1': [" +
+ " {" +
+ " 'core_node3': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n1'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node9': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n6'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node11': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n8'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63219_solr'," +
+ " 'isLive': true," +
+ " 'cores': 0.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {}" +
+ " }" +
+ " ]," +
+ " 'liveNodes': [" +
+ " '127.0.0.1:63191_solr'," +
+ " '127.0.0.1:63192_solr'," +
+ " '127.0.0.1:63219_solr'" +
+ " ]," +
+ " 'violations': [" +
+ " {" +
+ " 'collection': 'readApiTestViolations'," +
+ " 'shard': 'shard1'," +
+ " 'node': '127.0.0.1:63191_solr'," +
+ " 'violation': {" +
+ " 'replica': {'NRT': 3, 'count': 3}," +
+ " 'delta': 2.0" +
+ " }," +
+ " 'clause': {" +
+ " 'replica': '<3'," +
+ " 'shard': '#EACH'," +
+ " 'node': '#ANY'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'collection': 'readApiTestViolations'," +
+ " 'shard': 'shard1'," +
+ " 'node': '127.0.0.1:63192_solr'," +
+ " 'violation': {" +
+ " 'replica': {'NRT': 3, 'count': 3}," +
+ " 'delta': 2.0" +
+ " }," +
+ " 'clause': {" +
+ " 'replica': '<2'," +
+ " 'shard': '#EACH'," +
+ " 'node': '#ANY'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]," +
+ " 'config': {" +
+ " 'cluster-preferences': [" +
+ " {'minimize': 'cores', 'precision': 3}," +
+ " {'maximize': 'freedisk', 'precision': 100}," +
+ " {'minimize': 'sysLoadAvg', 'precision': 10}," +
+ " {'minimize': 'heapUsage', 'precision': 10}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " {'cores': '<10', 'node': '#ANY'}," +
+ " {'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole': 'overseer', 'replica': 0}" +
+ " ]" +
+ " }" +
+ " }}";
+ Map<String, Object> m = (Map<String, Object>) Utils.fromJSONString(diagnostics);
+
+ Policy policy = new Policy((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+ Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config")), cloudManagerFromDiagnostics);
+ assertEquals(2, suggestions.size());
+
+ }
+
+ static SolrCloudManager createCloudManagerFromDiagnostics(Map<String, Object> m) {
+ List<Map> sortedNodes = (List<Map>) Utils.getObjectByPath(m, false, "diagnostics/sortedNodes");
+ Set<String> liveNodes = new HashSet<>();
+ SolrClientNodeStateProvider nodeStateProvider = new SolrClientNodeStateProvider(null) {
+ @Override
+ protected void readReplicaDetails() {
+ for (Object o : sortedNodes) {
+ String node = (String) ((Map) o).get("node");
+ liveNodes.add(node);
+ Map nodeDetails = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, s -> new LinkedHashMap<>());
+ Map<String, Map<String, List<Map>>> replicas = (Map<String, Map<String, List<Map>>>) ((Map) o).get("replicas");
+ replicas.forEach((coll, shardVsReplicas) -> shardVsReplicas
+ .forEach((shard, repDetails) -> {
+ List<ReplicaInfo> reps = (List) ((Map) nodeDetails
+ .computeIfAbsent(coll, o1 -> new LinkedHashMap<>()))
+ .computeIfAbsent(shard, o12 -> new ArrayList<ReplicaInfo>());
+ for (Map map : repDetails) reps.add(new ReplicaInfo(map));
+ }));
+ }
+
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return nodeVsCollectionVsShardVsReplicaInfo.get(node) == null ?
+ Collections.emptyMap() :
+ nodeVsCollectionVsShardVsReplicaInfo.get(node);
+ }
+
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ for (Map n : sortedNodes) if (n.get("node").equals(node)) return n;
+ return Collections.emptyMap();
+ }
+ };
+ return new DelegatingCloudManager(null) {
+ @Override
+ public NodeStateProvider getNodeStateProvider() {
+ return nodeStateProvider;
+ }
+
+ @Override
+ public ClusterStateProvider getClusterStateProvider() {
+ return new DelegatingClusterStateProvider(null) {
+ @Override
+ public Set<String> getLiveNodes() {
+ return liveNodes;
+ }
+ };
+ }
+ };
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d087f9f1/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index a4c3b6d..d0cc4c5 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -49,6 +50,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
import org.junit.AfterClass;
import org.junit.Before;
@@ -352,4 +354,21 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
}
}
+ protected NamedList waitForResponse(Predicate<NamedList> predicate, SolrRequest request, int intervalInMillis, int numRetries, String messageOnFail) {
+ int i = 0;
+ for (; i < numRetries; i++) {
+ try {
+ NamedList<Object> response = cluster.getSolrClient().request(request);
+ if (predicate.test(response)) return response;
+ Thread.sleep(intervalInMillis);
+ } catch (RuntimeException rte) {
+ throw rte;
+ } catch (Exception e) {
+ throw new RuntimeException("error executing request", e);
+ }
+ }
+ fail("Tried " + i + " times , could not succeed. " + messageOnFail);
+ return null;
+ }
+
}