You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/05/25 23:52:04 UTC
[1/2] lucene-solr:feature/autoscaling: added extra clause
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling f0054a30d -> a0cd8decc
added extra clause
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a1f5cc8d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a1f5cc8d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a1f5cc8d
Branch: refs/heads/feature/autoscaling
Commit: a1f5cc8d9b5652038811461a93a16263a9207385
Parents: b933b60
Author: Noble Paul <no...@apache.org>
Authored: Fri May 26 09:21:48 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Fri May 26 09:21:48 2017 +0930
----------------------------------------------------------------------
.../apache/solr/cloud/autoscaling/Clause.java | 37 +++++++++++++-------
.../apache/solr/cloud/autoscaling/Policy.java | 15 ++++----
.../solr/cloud/autoscaling/TestPolicy.java | 19 +++++++---
3 files changed, 48 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a1f5cc8d/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
index 0406bc2..45cbe92 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
@@ -76,7 +76,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
int replicaCount = Integer.parseInt(String.valueOf(replica.val));
this.replica = new Condition(replica.name, replicaCount, replica.op);
} catch (NumberFormatException e) {
- throw new RuntimeException("Only an integer value is supported for replica "+Utils.toJSONString(m));
+ throw new RuntimeException("Only an integer value is supported for replica " + Utils.toJSONString(m));
}
m.forEach((s, o) -> parseCondition(s, o));
}
@@ -110,8 +110,8 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (v != 0) return v;
if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
- if(v ==0) {
- v = Integer.compare((Integer)this.replica.val, (Integer)that.replica.val);
+ if (v == 0) {
+ v = Integer.compare((Integer) this.replica.val, (Integer) that.replica.val);
v = this.replica.op == LESS_THAN ? v : v * -1;
}
return v;
@@ -195,16 +195,18 @@ public class Clause implements MapWriter, Comparable<Clause> {
final String shard, coll, node;
final Object actualVal;
final Integer delta;//how far is the actual value from the expected value
+ final Object tagKey;
private final int hash;
- private Violation(String coll, String shard, String node, Object actualVal, Integer delta ) {
+ private Violation(String coll, String shard, String node, Object actualVal, Integer delta, Object tagKey) {
this.shard = shard;
this.coll = coll;
this.node = node;
this.delta = delta;
this.actualVal = actualVal;
- hash = ("" + coll + " " + shard + " " + node + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
+ this.tagKey = tagKey;
+ hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
}
public Clause getClause() {
@@ -219,10 +221,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
@Override
public boolean equals(Object that) {
if (that instanceof Violation) {
- Violation ve = (Violation) that;
- return Objects.equals(this.shard, (ve).shard) &&
- Objects.equals(this.coll, (ve).coll) &&
- Objects.equals(this.node, (ve).node);
+ Violation v = (Violation) that;
+ return Objects.equals(this.shard, v.shard) &&
+ Objects.equals(this.coll, v.coll) &&
+ Objects.equals(this.node, v.node) &&
+ Objects.equals(this.tagKey, v.tagKey)
+ ;
}
return false;
}
@@ -232,6 +236,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
ew.putIfNotNull("collection", coll);
ew.putIfNotNull("shard", shard);
ew.putIfNotNull("node", node);
+ ew.putIfNotNull("tagKey", String.valueOf(tagKey));
ew.putIfNotNull("violation", (MapWriter) ew1 -> {
ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
String.valueOf(actualVal));
@@ -252,8 +257,14 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (!shard.isPass(shardVsCount.getKey())) continue;
for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
if (!replica.isPass(counts.getValue())) {
- errors.add(new Violation(e.getKey(), shardVsCount.getKey(),
- tag.name.equals("node") ? counts.getKey() : null, counts.getValue(), replica.delta(counts.getValue())));
+ errors.add(new Violation(
+ e.getKey(),
+ shardVsCount.getKey(),
+ tag.name.equals("node") ? counts.getKey() : null,
+ counts.getValue(),
+ replica.delta(counts.getValue()),
+ counts.getKey()
+ ));
}
}
}
@@ -261,7 +272,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
} else {
for (Row r : allRows) {
if (!tag.isPass(r)) {
- errors.add(new Violation(null, null, r.node, r.getVal(tag.name) , tag.delta(r.getVal(tag.name))));
+ errors.add(new Violation(null, null, r.node, r.getVal(tag.name), tag.delta(r.getVal(tag.name)), null));
}
}
}
@@ -285,7 +296,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
collMap.putIfAbsent(shardName, new HashMap<>());
Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
Object tagVal = row.getVal(tag.name);
- tagVsCount.putIfAbsent(tag.isPass(tagVal)? String.valueOf(tagVal) : "", new AtomicInteger());
+ tagVsCount.putIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", new AtomicInteger());
if (tag.isPass(tagVal)) {
tagVsCount.get(tagVal).addAndGet(shards.getValue().size());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a1f5cc8d/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index 98a10f3..dd3f7a8 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -384,13 +384,16 @@ public class Policy implements MapWriter {
boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
if (old == null || fresh.size() < old.size()) return true;
if(fresh.size() == old.size()){
- for (int i = 0; i < old.size(); i++) {
- if(fresh.get(i).equals(old.get(i))) {
- if (fresh.get(i) != null &&
- old.get(i).delta != null &&
- Math.abs(fresh.get(i).delta) < Math.abs(old.get(i).delta))
- return true;
+ for (int i = 0; i < fresh.size(); i++) {
+ Violation freshViolation = fresh.get(i);
+ Violation oldViolation = null;
+ for (Violation v : old) {
+ if(v.equals(freshViolation)){
+ oldViolation =v;
+ }
}
+ if (oldViolation != null && oldViolation.delta != null &&
+ Math.abs(fresh.get(i).delta) < Math.abs(oldViolation.delta)) return true;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a1f5cc8d/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 962ae15..64a54f3 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -221,7 +221,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
" 'cluster-policy':[" +
" {'cores':'<10','node':'#ANY'}," +
" {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
- " { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +
+ " { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +//greedy condition
" {'nodeRole':'overseer','replica':'0'}]," +
" 'cluster-preferences':[" +
" {'minimize':'cores', 'precision':3}," +
@@ -264,12 +264,23 @@ public class TestPolicy extends SolrTestCaseJ4 {
.getOperation();
assertNotNull(op);
assertEquals("node3", op.getParams().get("node"));
- op = suggester
+ suggester = suggester
+ .getSession()
+ .getSuggester(ADDREPLICA)
.hint(Hint.COLL, "newColl")
- .hint(Hint.SHARD, "shard1")
- .getOperation();
+ .hint(Hint.SHARD, "shard1");
+ op = suggester.getOperation();
assertNotNull(op);
assertEquals("node3", op.getParams().get("node"));
+
+ suggester = suggester
+ .getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals("node2", op.getParams().get("node"));
}
public void testMoveReplica() {
[2/2] lucene-solr:feature/autoscaling: Merge remote-tracking branch
'origin/feature/autoscaling' into feature/autoscaling
Posted by no...@apache.org.
Merge remote-tracking branch 'origin/feature/autoscaling' into feature/autoscaling
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a0cd8dec
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a0cd8dec
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a0cd8dec
Branch: refs/heads/feature/autoscaling
Commit: a0cd8decc6e6d11be4372c2cf9b044a285de14dc
Parents: a1f5cc8 f0054a3
Author: Noble Paul <no...@apache.org>
Authored: Fri May 26 09:21:58 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Fri May 26 09:21:58 2017 +0930
----------------------------------------------------------------------
.../org/apache/solr/cloud/ActionThrottle.java | 25 +-
.../org/apache/solr/cloud/ZkController.java | 2 +
.../solr/cloud/autoscaling/AutoScaling.java | 63 ++---
.../cloud/autoscaling/AutoScalingHandler.java | 7 +-
.../cloud/autoscaling/ComputePlanAction.java | 2 +-
.../cloud/autoscaling/ExecutePlanAction.java | 2 +-
.../solr/cloud/autoscaling/LogPlanAction.java | 2 +-
.../cloud/autoscaling/NodeAddedTrigger.java | 100 ++++----
.../solr/cloud/autoscaling/NodeLostTrigger.java | 94 ++++----
.../autoscaling/OverseerTriggerThread.java | 3 +-
.../cloud/autoscaling/ScheduledTriggers.java | 140 +++++++++--
.../solr/cloud/autoscaling/TriggerAction.java | 2 +-
.../solr/cloud/autoscaling/TriggerBase.java | 107 ++++++++
.../solr/cloud/autoscaling/TriggerEvent.java | 153 ++++++++++++
.../cloud/autoscaling/TriggerEventQueue.java | 100 ++++++++
.../src/java/org/apache/solr/util/IdUtils.java | 39 +++
.../java/org/apache/solr/util/TimeSource.java | 57 +++++
.../apache/solr/cloud/ActionThrottleTest.java | 21 +-
.../cloud/autoscaling/NodeAddedTriggerTest.java | 25 +-
.../cloud/autoscaling/NodeLostTriggerTest.java | 27 ++-
.../autoscaling/TriggerIntegrationTest.java | 241 ++++++++++++++++---
.../apache/solr/common/cloud/ZkStateReader.java | 2 +
.../java/org/apache/solr/common/util/Utils.java | 48 +++-
23 files changed, 1015 insertions(+), 247 deletions(-)
----------------------------------------------------------------------