You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/11/17 08:33:41 UTC
[20/40] lucene-solr:jira/solr-11285-sim: SOLR-11538: Implement port
suggestion
SOLR-11538: Implement port suggestion
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/65cd8bbb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/65cd8bbb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/65cd8bbb
Branch: refs/heads/jira/solr-11285-sim
Commit: 65cd8bbbf4208d7c4f99403b7d60dcbae2fc2c8e
Parents: fc678da
Author: Noble Paul <no...@apache.org>
Authored: Tue Nov 14 23:23:35 2017 +1030
Committer: Noble Paul <no...@apache.org>
Committed: Tue Nov 14 23:23:35 2017 +1030
----------------------------------------------------------------------
.../cloud/autoscaling/MoveReplicaSuggester.java | 6 ++-
.../solrj/cloud/autoscaling/Suggester.java | 10 ++++
.../solrj/cloud/autoscaling/Suggestion.java | 24 +++++----
.../solrj/cloud/autoscaling/Violation.java | 12 +++--
.../solrj/cloud/autoscaling/TestPolicy.java | 57 ++++++++++++++++++++
5 files changed, 95 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65cd8bbb/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
index 156f702..53b3bb7 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@ -51,14 +51,16 @@ public class MoveReplicaSuggester extends Suggester {
}
final int i = getMatrix().indexOf(fromRow);
- for (int j = getMatrix().size() - 1; j > i; j--) {
+ int stopAt = force ? 0 : i;
+ for (int j = getMatrix().size() - 1; j >= stopAt; j--) {
+ if (j == i) continue;
Row targetRow = getMatrix().get(j);
if(!targetRow.isLive) continue;
if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
targetRow = targetRow.addReplica(coll, shard, replicaInfo.getType());
List<Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getModifiedMatrix(getMatrix(), srcTmpRow, i), targetRow, j));
if (!containsNewErrors(errs) && isLessSerious(errs, leastSeriousViolation) &&
- Policy.compareRows(srcTmpRow, targetRow, session.getPolicy()) < 1) {
+ (force || Policy.compareRows(srcTmpRow, targetRow, session.getPolicy()) < 1)) {
leastSeriousViolation = errs;
targetNodeIndex = j;
sourceNodeIndex = i;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65cd8bbb/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index a0db4ed..d1e2031 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -47,6 +47,7 @@ public abstract class Suggester {
protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
Policy.Session session;
SolrRequest operation;
+ boolean force;
protected List<Violation> originalViolations = new ArrayList<>();
private boolean isInitialized = false;
@@ -65,6 +66,15 @@ public abstract class Suggester {
return this;
}
+ /**
+ * Normally, only less loaded nodes are used for moving replicas. If this is a violation and a MOVE must be performed,
+ * set the flag to true.
+ */
+ public Suggester forceOperation(boolean force) {
+ this.force = force;
+ return this;
+ }
+
abstract SolrRequest init();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65cd8bbb/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index c7c92d8..aec5f15 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -109,7 +109,7 @@ public class Suggestion {
COLL("collection", String.class, null, null, null),
SHARD("shard", String.class, null, null, null),
REPLICA("replica", Long.class, null, 0L, null),
- PORT(ImplicitSnitch.PORT, Long.class, null, 1L, 65535L),
+ PORT(ImplicitSnitch.PORT, Long.class, null, 1L, 65535L) ,
IP_1("ip_1", Long.class, null, 0L, 255L),
IP_2("ip_2", Long.class, null, 0L, 255L),
IP_3("ip_3", Long.class, null, 0L, 255L),
@@ -222,13 +222,7 @@ public class Suggestion {
@Override
public void getSuggestions(SuggestionCtx ctx) {
- if(ctx.violation == null)return;
- for (ReplicaInfoAndErr e : ctx.violation.getViolatingReplicas()) {
- Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
- .hint(Suggester.Hint.COLL_SHARD, new Pair<>( e.replicaInfo.getCollection(), e.replicaInfo.getShard()))
- .hint(Suggester.Hint.SRC_NODE, e.replicaInfo.getNode());
- if (ctx.addSuggestion(suggester) == null) break;
- }
+ perNodeSuggestions(ctx);
}
},;
@@ -254,7 +248,7 @@ public class Suggestion {
}
public void getSuggestions(SuggestionCtx ctx) {
-
+ perNodeSuggestions(ctx);
}
public void addViolatingReplicas(ViolationCtx ctx) {
@@ -262,6 +256,7 @@ public class Suggestion {
row.forEachReplica(replica -> {
if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
+ if(!ctx.currentViolation.matchShard(replica.getShard())) return;
if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
return;
ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
@@ -308,6 +303,17 @@ public class Suggestion {
}
}
+ private static void perNodeSuggestions(SuggestionCtx ctx) {
+ if (ctx.violation == null) return;
+ for (ReplicaInfoAndErr e : ctx.violation.getViolatingReplicas()) {
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .forceOperation(true)
+ .hint(Suggester.Hint.COLL_SHARD, new Pair<>(e.replicaInfo.getCollection(), e.replicaInfo.getShard()))
+ .hint(Suggester.Hint.SRC_NODE, e.replicaInfo.getNode());
+ if (ctx.addSuggestion(suggester) == null) break;
+ }
+ }
+
static {
for (Suggestion.ConditionType t : Suggestion.ConditionType.values()) Suggestion.validatetypes.put(t.tagName, t);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65cd8bbb/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 36266f5..105dea0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -34,7 +34,7 @@ public class Violation implements MapWriter {
final Object tagKey;
private final int hash;
private final Clause clause;
- private List<ReplicaInfoAndErr> violationVsMetaData = new ArrayList<>();
+ private List<ReplicaInfoAndErr> replicaInfoAndErrs = new ArrayList<>();
Violation(Clause clause, String coll, String shard, String node, Object actualVal, Long replicaCountDelta, Object tagKey) {
this.clause = clause;
@@ -48,17 +48,23 @@ public class Violation implements MapWriter {
}
public Violation addReplica(ReplicaInfoAndErr r) {
- violationVsMetaData.add(r);
+ replicaInfoAndErrs.add(r);
return this;
}
public List<ReplicaInfoAndErr> getViolatingReplicas() {
- return violationVsMetaData;
+ return replicaInfoAndErrs;
}
public Clause getClause() {
return clause;
}
+
+ public boolean matchShard(String shard) {
+ if (getClause().shard.op == Operand.WILDCARD) return true;
+ return this.shard == null || this.shard.equals(shard);
+ }
+
static class ReplicaInfoAndErr implements MapWriter{
final ReplicaInfo replicaInfo;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65cd8bbb/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 31e02f1..538f667 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -1521,4 +1521,61 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
}
+ public void testPortSuggestions() {
+ String autoScalingjson = "{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 3}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, shard:'#EACH', port : '8983'}" +
+ " ]" +
+ "}";
+
+
+ String dataproviderdata = "{" +
+ " 'liveNodes': [" +
+ " 'node1:8983'," +
+ " 'node2:8984'," +
+ " 'node3:8985'" +
+ " ]," +
+ " 'replicaInfo': {" +
+ " 'node1:8983': {" +
+ " 'c1': {" +
+ " 's1': [" +
+ " {'r1': {'type': 'NRT'}}," +
+ " {'r2': {'type': 'NRT'}}" +
+ " ]," +
+ " 's2': [" +
+ " {'r1': {'type': 'NRT'}}," +
+ " {'r2': {'type': 'NRT'}}" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " 'nodeValues': {" +
+ " 'node1:8983': {" +
+ " 'cores': 4," +
+ " 'freedisk': 334," +
+ " 'port': 8983" +
+ " }," +
+ " 'node2:8984': {" +
+ " 'cores': 0," +
+ " 'freedisk': 1000," +
+ " 'port': 8984" +
+ " }," +
+ " 'node3:8985': {" +
+ " 'cores': 0," +
+ " 'freedisk': 1500," +
+ " 'port': 8985" +
+ " }" +
+ " }" +
+ "}";
+ AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
+ List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
+ assertEquals(2, violations.size());
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
+ assertEquals(4, suggestions.size());
+ }
+
}