You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2018/07/20 09:00:58 UTC
lucene-solr:jira/solr-11990: SOLR-11990: Added WITH_COLLECTION to
global tags. Fixed implementation of addViolatingReplicas and getSuggestions
in the clause impl. Added more asserts in testWithCollectionSuggestions.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11990 dbadb3321 -> 4824933fd
SOLR-11990: Added WITH_COLLECTION to global tags. Fixed implementation of addViolatingReplicas and getSuggestions in the clause impl. Added more asserts in testWithCollectionSuggestions.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4824933f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4824933f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4824933f
Branch: refs/heads/jira/solr-11990
Commit: 4824933fd6eb7d1773acbff1a1a0c5e670226e0b
Parents: dbadb33
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Jul 20 14:30:52 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Jul 20 14:30:52 2018 +0530
----------------------------------------------------------------------
.../client/solrj/cloud/autoscaling/Clause.java | 2 +-
.../client/solrj/cloud/autoscaling/Policy.java | 3 +-
.../solrj/cloud/autoscaling/Suggestion.java | 45 ++++++++++++++------
.../solrj/cloud/autoscaling/Violation.java | 2 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 22 ++++++++++
5 files changed, 57 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4824933f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index b705975..aeced28 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -335,7 +335,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
for (Row r : session.matrix) {
SealedClause sealedClause = getSealedClause(computedValueEvaluator);
if (!sealedClause.getGlobalTag().isPass(r)) {
- ConditionType.CORES.addViolatingReplicas(ctx.reset(null, null,
+ sealedClause.getGlobalTag().varType.addViolatingReplicas(ctx.reset(null, null,
new Violation(sealedClause, null, null, r.node, r.getVal(sealedClause.globalTag.name), sealedClause.globalTag.delta(r.getVal(globalTag.name)), null)));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4824933f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 6cd31b8..e07a47f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -44,6 +44,7 @@ import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
@@ -76,7 +77,7 @@ public class Policy implements MapWriter {
public static final String POLICIES = "policies";
public static final String CLUSTER_POLICY = "cluster-policy";
public static final String CLUSTER_PREFERENCES = "cluster-preferences";
- public static final Set<String> GLOBAL_ONLY_TAGS = Collections.singleton("cores");
+ public static final Set<String> GLOBAL_ONLY_TAGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("cores", CollectionAdminParams.WITH_COLLECTION)));
public static final List<Preference> DEFAULT_PREFERENCES = Collections.unmodifiableList(
Arrays.asList(
new Preference((Map<String, Object>) Utils.fromJSONString("{minimize : cores, precision:1}")),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4824933f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 050421a..880808a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -37,7 +37,6 @@ import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.V2RequestSupport;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Clause.ComputedType;
import org.apache.solr.client.solrj.cloud.autoscaling.Violation.ReplicaInfoAndErr;
import org.apache.solr.common.cloud.Replica;
@@ -557,15 +556,24 @@ public class Suggestion {
*/
@Override
public void addViolatingReplicas(ViolationCtx ctx) {
- String collectionName = ctx.currentViolation.coll;
String node = ctx.currentViolation.node;
for (Row row : ctx.allRows) {
if (node.equals(row.node)) {
- row.forEachReplica(r -> {
- if (collectionName.equals(r.getCollection())) {
- ctx.currentViolation.addReplica(new ReplicaInfoAndErr(r).withDelta(1.0d));
- }
- });
+ Map<String, String> withCollectionMap = (Map<String, String>) row.getVal(WITH_COLLECTION.tagName);
+ if (withCollectionMap != null) {
+ row.forEachReplica(r -> {
+ String withCollection = withCollectionMap.get(r.getCollection());
+ if (withCollection != null) {
+ // test whether this row has at least 1 replica of withCollection, else there is a violation
+ Set<String> uniqueCollections = new HashSet<>();
+ row.forEachReplica(replicaInfo -> uniqueCollections.add(replicaInfo.getCollection()));
+ if (!uniqueCollections.contains(withCollection)) {
+ ctx.currentViolation.addReplica(new ReplicaInfoAndErr(r).withDelta(1.0d));
+ }
+ }
+ });
+ ctx.currentViolation.replicaCountDelta = (double) ctx.currentViolation.getViolatingReplicas().size();
+ }
}
}
}
@@ -577,18 +585,27 @@ public class Suggestion {
@Override
public void getSuggestions(SuggestionCtx ctx) {
+ if (ctx.violation.getViolatingReplicas().isEmpty()) return;
+
Map<String, Object> nodeValues = ctx.session.nodeStateProvider.getNodeValues(ctx.violation.node, Collections.singleton(WITH_COLLECTION.tagName));
Map<String, String> withCollectionsMap = (Map<String, String>) nodeValues.get(WITH_COLLECTION.tagName);
if (withCollectionsMap == null) return;
- String withCollection = withCollectionsMap.get(ctx.violation.coll);
- if (withCollection == null) return;
+ Set<String> uniqueCollections = new HashSet<>();
+ for (ReplicaInfoAndErr replicaInfoAndErr : ctx.violation.getViolatingReplicas()) {
+ uniqueCollections.add(replicaInfoAndErr.replicaInfo.getCollection());
+ }
+
+ for (String collection : uniqueCollections) {
+ String withCollection = withCollectionsMap.get(collection);
+ if (withCollection == null) continue;
- Suggester suggester = ctx.session.getSuggester(ADDREPLICA)
- .forceOperation(true)
- .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
- .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
- ctx.addSuggestion(suggester);
+ Suggester suggester = ctx.session.getSuggester(ADDREPLICA)
+ .forceOperation(true)
+ .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
+ .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
+ ctx.addSuggestion(suggester);
+ }
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4824933f/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index d4ab0e6..7b0f0f3 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -30,7 +30,7 @@ import org.apache.solr.common.util.Utils;
public class Violation implements MapWriter {
final String shard, coll, node;
final Object actualVal;
- final Double replicaCountDelta;//how many extra replicas
+ Double replicaCountDelta;//how many extra replicas
final Object tagKey;
private final int hash;
private final Clause clause;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4824933f/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index fb3db53..f766ad5 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -389,6 +389,28 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
});
assertNotNull(l);
+ assertEquals(2, l.size());
+
+ // collect the set of nodes to which replicas are being added
+ Set<String> nodes = new HashSet<>(2);
+
+ m = l.get(0).toMap(new LinkedHashMap<>());
+ assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
+ assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
+ assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(m, true, "operation/path"));
+ assertNotNull(Utils.getObjectByPath(m, false, "operation/command/add-replica"));
+ nodes.add((String) Utils.getObjectByPath(m, true, "operation/command/add-replica/node"));
+
+ m = l.get(1).toMap(new LinkedHashMap<>());
+ assertEquals(1.0d, Utils.getObjectByPath(m, true, "violation/violation/delta"));
+ assertEquals("POST", Utils.getObjectByPath(m, true, "operation/method"));
+ assertEquals("/c/articles_coll/shards", Utils.getObjectByPath(m, true, "operation/path"));
+ assertNotNull(Utils.getObjectByPath(m, false, "operation/command/add-replica"));
+ nodes.add((String) Utils.getObjectByPath(m, true, "operation/command/add-replica/node"));
+
+ assertEquals(2, nodes.size());
+ assertTrue(nodes.contains("node1"));
+ assertTrue(nodes.contains("node2"));
}
public void testValidate() {