You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/05/25 08:25:16 UTC
[1/2] lucene-solr:feature/autoscaling: fixed and added test for
greedy conditions
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling f9cf54963 -> b933b6040
fixed and added test for greedy conditions
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9ad811b3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9ad811b3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9ad811b3
Branch: refs/heads/feature/autoscaling
Commit: 9ad811b36f3538fc8e1beb0c1b1c438a3ca4db84
Parents: fe21766
Author: Noble Paul <no...@apache.org>
Authored: Thu May 25 17:55:00 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Thu May 25 17:55:00 2017 +0930
----------------------------------------------------------------------
.../cloud/autoscaling/AddReplicaSuggester.java | 21 ++-
.../apache/solr/cloud/autoscaling/Clause.java | 47 ++++--
.../apache/solr/cloud/autoscaling/Operand.java | 36 ++++-
.../apache/solr/cloud/autoscaling/Policy.java | 39 ++++-
.../solr/cloud/autoscaling/TestPolicy.java | 149 +++++++++++++------
5 files changed, 217 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ad811b3/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
index d0e510d..ac17ac2 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
@@ -38,6 +38,8 @@ class AddReplicaSuggester extends Suggester {
throw new RuntimeException("add-replica requires 'collection' and 'shard'");
//iterate through elements and identify the least loaded
+ List<Clause.Violation> leastSeriousViolation = null;
+ Integer targetNodeIndex = null;
for (int i = getMatrix().size() - 1; i >= 0; i--) {
Row row = getMatrix().get(i);
if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
@@ -45,14 +47,21 @@ class AddReplicaSuggester extends Suggester {
tmpRow.violations.clear();
List<Clause.Violation> errs = testChangedRow(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
-
- if (!containsNewErrors(errs)) {// there are no rule violations
- getMatrix().set(i, getMatrix().get(i).addReplica(coll, shard));
- return CollectionAdminRequest
- .addReplicaToShard(coll, shard)
- .setNode(row.node);
+ if(!containsNewErrors(errs)) {
+ if(isLessSerious(errs, leastSeriousViolation)){
+ leastSeriousViolation = errs;
+ targetNodeIndex = i;
+ }
}
}
+
+ if (targetNodeIndex != null) {// there are no rule violations
+ getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
+ return CollectionAdminRequest
+ .addReplicaToShard(coll, shard)
+ .setNode(getMatrix().get(targetNodeIndex).node);
+ }
+
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ad811b3/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
index d466ce9..0406bc2 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
@@ -71,7 +71,13 @@ public class Clause implements MapWriter, Comparable<Clause> {
} else {
collection = parse(COLLECTION, m);
shard = parse(SHARD, m);
- this.replica = parse(REPLICA, m);
+ Condition replica = parse(REPLICA, m);
+ try {
+ int replicaCount = Integer.parseInt(String.valueOf(replica.val));
+ this.replica = new Condition(replica.name, replicaCount, replica.op);
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Only an integer value is supported for replica "+Utils.toJSONString(m));
+ }
m.forEach((s, o) -> parseCondition(s, o));
}
if (tag == null)
@@ -102,12 +108,17 @@ public class Clause implements MapWriter, Comparable<Clause> {
try {
int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
if (v != 0) return v;
- return this.isPerCollectiontag() && that.isPerCollectiontag() ?
- Integer.compare(this.replica.op.priority, that.replica.op.priority) :
- 0;
+ if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
+ v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
+ if(v ==0) {
+ v = Integer.compare((Integer)this.replica.val, (Integer)that.replica.val);
+ v = this.replica.op == LESS_THAN ? v : v * -1;
+ }
+ return v;
+ } else {
+ return 0;
+ }
} catch (NullPointerException e) {
- System.out.println("this: " + Utils.toJSONString(this));
- System.out.println("thAt: " + Utils.toJSONString(that));
throw e;
}
}
@@ -147,6 +158,10 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
return false;
}
+
+ public Integer delta(Object val) {
+ return op.delta(this.val, val);
+ }
}
static Condition parse(String s, Map m) {
@@ -179,13 +194,15 @@ public class Clause implements MapWriter, Comparable<Clause> {
public class Violation implements MapWriter {
final String shard, coll, node;
final Object actualVal;
+ final Integer delta;//how far is the actual value from the expected value
private final int hash;
- private Violation(String coll, String shard, String node, Object actualVal) {
+ private Violation(String coll, String shard, String node, Object actualVal, Integer delta ) {
this.shard = shard;
this.coll = coll;
this.node = node;
+ this.delta = delta;
this.actualVal = actualVal;
hash = ("" + coll + " " + shard + " " + node + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
}
@@ -215,8 +232,11 @@ public class Clause implements MapWriter, Comparable<Clause> {
ew.putIfNotNull("collection", coll);
ew.putIfNotNull("shard", shard);
ew.putIfNotNull("node", node);
- ew.putIfNotNull("violation", (MapWriter) ew1 -> ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
- String.valueOf(actualVal)));
+ ew.putIfNotNull("violation", (MapWriter) ew1 -> {
+ ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
+ String.valueOf(actualVal));
+ ew1.putIfNotNull("delta", delta);
+ });
ew.put("clause", getClause());
}
}
@@ -233,7 +253,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
if (!replica.isPass(counts.getValue())) {
errors.add(new Violation(e.getKey(), shardVsCount.getKey(),
- tag.name.equals("node") ? counts.getKey() : null, counts.getValue()));
+ tag.name.equals("node") ? counts.getKey() : null, counts.getValue(), replica.delta(counts.getValue())));
}
}
}
@@ -241,7 +261,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
} else {
for (Row r : allRows) {
if (!tag.isPass(r)) {
- errors.add(new Violation(null, null, r.node, r.getVal(tag.name) ));
+ errors.add(new Violation(null, null, r.node, r.getVal(tag.name) , tag.delta(r.getVal(tag.name))));
}
}
}
@@ -264,11 +284,10 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (!shard.isPass(shardName)) break;
collMap.putIfAbsent(shardName, new HashMap<>());
Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
- AtomicInteger count = null;
Object tagVal = row.getVal(tag.name);
+ tagVsCount.putIfAbsent(tag.isPass(tagVal)? String.valueOf(tagVal) : "", new AtomicInteger());
if (tag.isPass(tagVal)) {
- tagVsCount.put(String.valueOf(tagVal), count = tagVsCount.getOrDefault(tagVal, new AtomicInteger()));
- count.addAndGet(shards.getValue().size());
+ tagVsCount.get(tagVal).addAndGet(shards.getValue().size());
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ad811b3/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
index 6ff1ef5..5371c25 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
@@ -40,12 +40,23 @@ public enum Operand {
return ANY.equals(val) || Policy.EACH.equals(val) ? val : null;
}
},
- EQUAL("", 0),
+ EQUAL("", 0) {
+ @Override
+ public int _delta(int expected, int actual) {
+ return expected - actual;
+ }
+ },
NOT_EQUAL("!", 2) {
@Override
public TestStatus match(Object ruleVal, Object testVal) {
return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
}
+
+ @Override
+ public int _delta(int expected, int actual) {
+ return expected - actual;
+ }
+
},
GREATER_THAN(">", 1) {
@Override
@@ -60,6 +71,10 @@ public enum Operand {
return compareNum(ruleVal, testVal) == 1 ? PASS : FAIL;
}
+ @Override
+ protected int _delta(int expected, int actual) {
+ return actual > expected ? 0 : (expected + 1) - actual;
+ }
},
LESS_THAN("<", 2) {
@Override
@@ -69,6 +84,11 @@ public enum Operand {
}
@Override
+ protected int _delta(int expected, int actual) {
+ return actual < expected ? 0 : (expected ) - actual;
+ }
+
+ @Override
public Object parse(String val) {
return checkNumeric(super.parse(val));
}
@@ -118,4 +138,18 @@ public enum Operand {
}
return o;
}
+
+ public Integer delta(Object expected, Object actual) {
+ try {
+ Integer expectedInt = Integer.parseInt(String.valueOf(expected));
+ Integer actualInt = Integer.parseInt(String.valueOf(actual));
+ return _delta(expectedInt, actualInt);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ protected int _delta(int expected, int actual) {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ad811b3/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index 614bfa2..98a10f3 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -348,13 +348,20 @@ public class Policy implements MapWriter {
public SolrRequest getOperation() {
if (!isInitialized) {
String coll = (String) hints.get(Hint.COLL);
- if(coll != null){
- // if this is not a known collection from the existing clusterstate,
- // then add it
- if(session.matrix.stream().noneMatch(row -> row.replicaInfo.containsKey(coll))){
- session.matrix.get(0).replicaInfo.put(coll, new HashMap<>());
- session.addClausesForCollection(session.dataProvider, coll);
- Collections.sort(session.expandedClauses);
+ String shard = (String) hints.get(Hint.SHARD);
+ // if this is not a known collection from the existing clusterstate,
+ // then add it
+ if(session.matrix.stream().noneMatch(row -> row.replicaInfo.containsKey(coll))){
+ session.addClausesForCollection(session.dataProvider, coll);
+ Collections.sort(session.expandedClauses);
+ }
+ if(coll != null) {
+ for (Row row : session.matrix) {
+ if (!row.replicaInfo.containsKey(coll)) row.replicaInfo.put(coll, new HashMap<>());
+ if(shard != null){
+ Map<String, List<ReplicaInfo>> shardInfo = row.replicaInfo.get(coll);
+ if(!shardInfo.containsKey(shard)) shardInfo.put(shard, new ArrayList<>());
+ }
}
}
session.applyRules();
@@ -373,7 +380,23 @@ public class Policy implements MapWriter {
return session.matrix;
}
- boolean containsNewErrors(List<Clause.Violation> errs){
+
+ boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
+ if (old == null || fresh.size() < old.size()) return true;
+ if(fresh.size() == old.size()){
+ for (int i = 0; i < old.size(); i++) {
+ if(fresh.get(i).equals(old.get(i))) {
+ if (fresh.get(i) != null &&
+ old.get(i).delta != null &&
+ Math.abs(fresh.get(i).delta) < Math.abs(old.get(i).delta))
+ return true;
+ }
+ }
+
+ }
+ return false;
+ }
+ boolean containsNewErrors(List<Violation> errs){
for (Clause.Violation err : errs) {
if(!originalViolations.contains(err)) return true;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ad811b3/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 36c18e0..962ae15 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -55,7 +55,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertFalse(c.replica.isPass(2));
assertFalse(c.replica.isPass(1));
- c = new Clause((Map<String, Object>) Utils.fromJSONString("{nodeRole:'!overseer'}"));
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}"));
assertTrue(c.tag.isPass("OVERSEER"));
assertFalse(c.tag.isPass("overseer"));
}
@@ -96,12 +96,12 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("1", String.valueOf(clauses.get(0).original.get("replica")));
assertEquals("0", String.valueOf(clauses.get(1).original.get("replica")));
assertEquals("#ANY", clauses.get(3).original.get("shard"));
- assertEquals("rack1",clauses.get(2).original.get("rack"));
+ assertEquals("rack1", clauses.get(2).original.get("rack"));
assertEquals("overseer", clauses.get(1).original.get("nodeRole"));
}
- public void testConditionsSort(){
+ public void testConditionsSort() {
String rules = "{" +
" 'cluster-policy':[" +
" { 'nodeRole':'overseer', replica: 0, 'strict':false}," +
@@ -112,9 +112,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
Policy p = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
List<Clause> clauses = new ArrayList<>(p.getClusterPolicy());
Collections.sort(clauses);
- assertEquals("nodeRole", clauses.get(0).tag.name);
- assertEquals("rack", clauses.get(1).tag.name);
+ assertEquals("nodeRole", clauses.get(1).tag.name);
+ assertEquals("rack", clauses.get(0).tag.name);
}
+
public static String clusterState = "{'gettingstarted':{" +
" 'router':{'name':'compositeId'}," +
" 'shards':{" +
@@ -188,8 +189,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Violation> violations = session.getViolations();
assertEquals(3, violations.size());
- assertTrue( violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.val)));
- assertTrue( violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.name)));
+ assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.val)));
+ assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.name)));
assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.op == Operand.LESS_THAN && "node".equals(violation.getClause().tag.name))));
Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
@@ -214,8 +215,64 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
-
- public void testMoveReplica(){
+
+ public void testGreedyConditions() {
+ String autoscaleJson = "{" +
+ " 'cluster-policy':[" +
+ " {'cores':'<10','node':'#ANY'}," +
+ " {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
+ " { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +
+ " {'nodeRole':'overseer','replica':'0'}]," +
+ " 'cluster-preferences':[" +
+ " {'minimize':'cores', 'precision':3}," +
+ " {'maximize':'freedisk','precision':100}]}";
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "}");
+
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
+ ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
+ ClusterDataProvider cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return clusterDataProvider.getNodeValues(node, tags);
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return clusterDataProvider.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return clusterDataProvider.getNodes();
+ }
+
+ @Override
+ public String getPolicy(String coll) {
+ return null;
+ }
+ };
+ Policy.Session session = policy.createSession(cdp);
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
+ SolrRequest op = suggester
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1")
+ .getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+ op = suggester
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1")
+ .getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+ }
+
+ public void testMoveReplica() {
String autoscaleJson = "{" +
" 'cluster-policy':[" +
" {'cores':'<10','node':'#ANY'}," +
@@ -232,17 +289,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
" {'core_node2':{}}]}}}");
Map m = (Map) Utils.getObjectByPath(replicaInfoMap, false, "127.0.0.1:60089_solr/compute_plan_action_test");
m.put("shard1", Arrays.asList(
- new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Collections.emptyMap()),
- new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Collections.emptyMap())
- ));
-
- Map<String, Map<String,Object>> tagsMap = (Map) Utils.fromJSONString( "{" +
- " '127.0.0.1:60099_solr':{" +
- " 'cores':0," +
- " 'freedisk':918005641216}," +
- " '127.0.0.1:60089_solr':{" +
- " 'cores':2," +
- " 'freedisk':918005641216}}}");
+ new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Collections.emptyMap()),
+ new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Collections.emptyMap())
+ ));
+
+ Map<String, Map<String, Object>> tagsMap = (Map) Utils.fromJSONString("{" +
+ " '127.0.0.1:60099_solr':{" +
+ " 'cores':0," +
+ " 'freedisk':918005641216}," +
+ " '127.0.0.1:60089_solr':{" +
+ " 'cores':2," +
+ " 'freedisk':918005641216}}}");
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
Policy.Session session = policy.createSession(new ClusterDataProvider() {
@@ -268,7 +325,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
});
Policy.Suggester suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA)
- .hint(Policy.Suggester.Hint.TARGET_NODE, "127.0.0.1:60099_solr");
+ .hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
SolrParams op = suggester.getOperation().getParams();
assertNotNull(op);
session = suggester.getSession();
@@ -277,7 +334,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(op);
}
- public void testOtherTag(){
+ public void testOtherTag() {
String rules = "{" +
"'cluster-preferences':[" +
"{'minimize':'cores','precision':2}," +
@@ -285,13 +342,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
"{'minimize':'heapUsage','precision':1000}" +
"]," +
"'cluster-policy':[" +
- "{'nodeRole':'!overseer','strict':false}," +
+ "{replica:0, 'nodeRole':'overseer','strict':false}," +
"{'replica':'<1','node':'node3'}," +
"{'replica':'<2','node':'#ANY','shard':'#EACH'}" +
"]," +
"'policies':{" +
"'p1':[" +
- "{'nodeRole':'!overseer','strict':false}," +
+ "{replica:0, 'nodeRole':'overseer','strict':false}," +
"{'replica':'<1','node':'node3'}," +
"{'replica':'<2','node':'#ANY','shard':'#EACH'}," +
"{'replica':'<3','shard':'#EACH','rack':'#ANY'}" +
@@ -328,7 +385,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
return "p1";
}
};
- Policy.Session session = policy.createSession(cdp);
+ Policy.Session session = policy.createSession(cdp);
CollectionAdminRequest.AddReplica op = (CollectionAdminRequest.AddReplica) session
.getSuggester(ADDREPLICA)
@@ -339,19 +396,19 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
- private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
+ private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
return new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- Map<String, Object> result = new LinkedHashMap<>();
- tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
- return result;
- }
-
- @Override
- public Collection<String> getNodes() {
- return nodeValues.keySet();
- }
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return nodeValues.keySet();
+ }
@Override
public String getPolicy(String coll) {
@@ -359,15 +416,15 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return getReplicaDetails(node, clusterState);
- }
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return getReplicaDetails(node, clusterState);
+ }
- };
+ };
}
public void testMultiReplicaPlacement() {
- String autoScaleJson ="{" +
+ String autoScaleJson = "{" +
" 'cluster-preferences': [" +
" { maximize : freedisk , precision: 50}," +
" { minimize : cores, precision: 2}" +
@@ -387,7 +444,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}";
- Map<String,Map> nodeValues = (Map<String, Map>) Utils.fromJSONString( "{" +
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:12, freedisk: 334, heap:10480, rack:rack3}," +
"node2:{cores:4, freedisk: 749, heap:6873, sysprop.fs : ssd, rack:rack1}," +
"node3:{cores:7, freedisk: 262, heap:7834, rack:rack4}," +
@@ -420,8 +477,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
"newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
dataProvider, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3);
- assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2","node1","node3")));
- assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2","node1","node3")));
+ assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2", "node1", "node3")));
+ assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2", "node1", "node3")));
}
@@ -443,7 +500,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
if (shardVsReplicaStats == null) result.put(collName, shardVsReplicaStats = new HashMap<>());
List<Policy.ReplicaInfo> replicaInfos = shardVsReplicaStats.get(shard);
if (replicaInfos == null) shardVsReplicaStats.put(shard, replicaInfos = new ArrayList<>());
- replicaInfos.add(new Policy.ReplicaInfo(replicaName,collName, shard, new HashMap<>()));
+ replicaInfos.add(new Policy.ReplicaInfo(replicaName, collName, shard, new HashMap<>()));
});
});
});
[2/2] lucene-solr:feature/autoscaling: Merge remote-tracking branch
'origin/feature/autoscaling' into feature/autoscaling
Posted by no...@apache.org.
Merge remote-tracking branch 'origin/feature/autoscaling' into feature/autoscaling
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b933b604
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b933b604
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b933b604
Branch: refs/heads/feature/autoscaling
Commit: b933b604070012700d5d2b89cb920214b0b6e355
Parents: 9ad811b f9cf549
Author: Noble Paul <no...@apache.org>
Authored: Thu May 25 17:55:10 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Thu May 25 17:55:10 2017 +0930
----------------------------------------------------------------------
.../solr/cloud/autoscaling/AutoScaling.java | 6 ++
.../cloud/autoscaling/NodeAddedTrigger.java | 11 +++
.../solr/cloud/autoscaling/NodeLostTrigger.java | 11 +++
.../cloud/autoscaling/ScheduledTriggers.java | 5 +-
.../cloud/autoscaling/NodeAddedTriggerTest.java | 62 ++++++++++++++
.../cloud/autoscaling/NodeLostTriggerTest.java | 75 +++++++++++++++-
.../autoscaling/TriggerIntegrationTest.java | 90 +++++++++++++++-----
7 files changed, 234 insertions(+), 26 deletions(-)
----------------------------------------------------------------------