You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/05/25 23:52:04 UTC

[1/2] lucene-solr:feature/autoscaling: added extra clause

Repository: lucene-solr
Updated Branches:
  refs/heads/feature/autoscaling f0054a30d -> a0cd8decc


added extra clause


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a1f5cc8d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a1f5cc8d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a1f5cc8d

Branch: refs/heads/feature/autoscaling
Commit: a1f5cc8d9b5652038811461a93a16263a9207385
Parents: b933b60
Author: Noble Paul <no...@apache.org>
Authored: Fri May 26 09:21:48 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Fri May 26 09:21:48 2017 +0930

----------------------------------------------------------------------
 .../apache/solr/cloud/autoscaling/Clause.java   | 37 +++++++++++++-------
 .../apache/solr/cloud/autoscaling/Policy.java   | 15 ++++----
 .../solr/cloud/autoscaling/TestPolicy.java      | 19 +++++++---
 3 files changed, 48 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a1f5cc8d/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
index 0406bc2..45cbe92 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
@@ -76,7 +76,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
         int replicaCount = Integer.parseInt(String.valueOf(replica.val));
         this.replica = new Condition(replica.name, replicaCount, replica.op);
       } catch (NumberFormatException e) {
-        throw new RuntimeException("Only an integer value is supported for replica "+Utils.toJSONString(m));
+        throw new RuntimeException("Only an integer value is supported for replica " + Utils.toJSONString(m));
       }
       m.forEach((s, o) -> parseCondition(s, o));
     }
@@ -110,8 +110,8 @@ public class Clause implements MapWriter, Comparable<Clause> {
       if (v != 0) return v;
       if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
         v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
-        if(v ==0) {
-          v = Integer.compare((Integer)this.replica.val, (Integer)that.replica.val);
+        if (v == 0) {
+          v = Integer.compare((Integer) this.replica.val, (Integer) that.replica.val);
           v = this.replica.op == LESS_THAN ? v : v * -1;
         }
         return v;
@@ -195,16 +195,18 @@ public class Clause implements MapWriter, Comparable<Clause> {
     final String shard, coll, node;
     final Object actualVal;
     final Integer delta;//how far is the actual value from the expected value
+    final Object tagKey;
     private final int hash;
 
 
-    private Violation(String coll, String shard, String node, Object actualVal, Integer delta ) {
+    private Violation(String coll, String shard, String node, Object actualVal, Integer delta, Object tagKey) {
       this.shard = shard;
       this.coll = coll;
       this.node = node;
       this.delta = delta;
       this.actualVal = actualVal;
-      hash = ("" + coll + " " + shard + " " + node + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
+      this.tagKey = tagKey;
+      hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
     }
 
     public Clause getClause() {
@@ -219,10 +221,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
     @Override
     public boolean equals(Object that) {
       if (that instanceof Violation) {
-        Violation ve = (Violation) that;
-        return Objects.equals(this.shard, (ve).shard) &&
-            Objects.equals(this.coll, (ve).coll) &&
-            Objects.equals(this.node, (ve).node);
+        Violation v = (Violation) that;
+        return Objects.equals(this.shard, v.shard) &&
+            Objects.equals(this.coll, v.coll) &&
+            Objects.equals(this.node, v.node) &&
+            Objects.equals(this.tagKey, v.tagKey)
+            ;
       }
       return false;
     }
@@ -232,6 +236,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
       ew.putIfNotNull("collection", coll);
       ew.putIfNotNull("shard", shard);
       ew.putIfNotNull("node", node);
+      ew.putIfNotNull("tagKey", String.valueOf(tagKey));
       ew.putIfNotNull("violation", (MapWriter) ew1 -> {
         ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
             String.valueOf(actualVal));
@@ -252,8 +257,14 @@ public class Clause implements MapWriter, Comparable<Clause> {
           if (!shard.isPass(shardVsCount.getKey())) continue;
           for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
             if (!replica.isPass(counts.getValue())) {
-              errors.add(new Violation(e.getKey(), shardVsCount.getKey(),
-                  tag.name.equals("node") ? counts.getKey() : null, counts.getValue(), replica.delta(counts.getValue())));
+              errors.add(new Violation(
+                  e.getKey(),
+                  shardVsCount.getKey(),
+                  tag.name.equals("node") ? counts.getKey() : null,
+                  counts.getValue(),
+                  replica.delta(counts.getValue()),
+                  counts.getKey()
+              ));
             }
           }
         }
@@ -261,7 +272,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
     } else {
       for (Row r : allRows) {
         if (!tag.isPass(r)) {
-          errors.add(new Violation(null, null, r.node, r.getVal(tag.name) , tag.delta(r.getVal(tag.name))));
+          errors.add(new Violation(null, null, r.node, r.getVal(tag.name), tag.delta(r.getVal(tag.name)), null));
         }
       }
     }
@@ -285,7 +296,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
           collMap.putIfAbsent(shardName, new HashMap<>());
           Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
           Object tagVal = row.getVal(tag.name);
-          tagVsCount.putIfAbsent(tag.isPass(tagVal)? String.valueOf(tagVal) : "", new AtomicInteger());
+          tagVsCount.putIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", new AtomicInteger());
           if (tag.isPass(tagVal)) {
             tagVsCount.get(tagVal).addAndGet(shards.getValue().size());
           }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a1f5cc8d/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index 98a10f3..dd3f7a8 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -384,13 +384,16 @@ public class Policy implements MapWriter {
     boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
       if (old == null || fresh.size() < old.size()) return true;
       if(fresh.size() == old.size()){
-        for (int i = 0; i < old.size(); i++) {
-          if(fresh.get(i).equals(old.get(i))) {
-            if (fresh.get(i) != null &&
-                old.get(i).delta != null &&
-                Math.abs(fresh.get(i).delta) < Math.abs(old.get(i).delta))
-              return true;
+        for (int i = 0; i < fresh.size(); i++) {
+          Violation freshViolation = fresh.get(i);
+          Violation oldViolation = null;
+          for (Violation v : old) {
+            if(v.equals(freshViolation)){
+              oldViolation =v;
+            }
           }
+          if (oldViolation != null && oldViolation.delta != null &&
+              Math.abs(fresh.get(i).delta) < Math.abs(oldViolation.delta)) return true;
         }
 
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a1f5cc8d/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 962ae15..64a54f3 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -221,7 +221,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
         "      'cluster-policy':[" +
         "      {'cores':'<10','node':'#ANY'}," +
         "      {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
-        "      { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +
+        "      { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +//greedy condition
         "      {'nodeRole':'overseer','replica':'0'}]," +
         "      'cluster-preferences':[" +
         "      {'minimize':'cores', 'precision':3}," +
@@ -264,12 +264,23 @@ public class TestPolicy extends SolrTestCaseJ4 {
         .getOperation();
     assertNotNull(op);
     assertEquals("node3", op.getParams().get("node"));
-    op = suggester
+    suggester = suggester
+        .getSession()
+        .getSuggester(ADDREPLICA)
         .hint(Hint.COLL, "newColl")
-        .hint(Hint.SHARD, "shard1")
-        .getOperation();
+        .hint(Hint.SHARD, "shard1");
+    op = suggester.getOperation();
     assertNotNull(op);
     assertEquals("node3", op.getParams().get("node"));
+
+    suggester = suggester
+        .getSession()
+        .getSuggester(ADDREPLICA)
+        .hint(Hint.COLL, "newColl")
+        .hint(Hint.SHARD, "shard1");
+    op = suggester.getOperation();
+    assertNotNull(op);
+    assertEquals("node2", op.getParams().get("node"));
   }
 
   public void testMoveReplica() {


[2/2] lucene-solr:feature/autoscaling: Merge remote-tracking branch 'origin/feature/autoscaling' into feature/autoscaling

Posted by no...@apache.org.
Merge remote-tracking branch 'origin/feature/autoscaling' into feature/autoscaling


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a0cd8dec
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a0cd8dec
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a0cd8dec

Branch: refs/heads/feature/autoscaling
Commit: a0cd8decc6e6d11be4372c2cf9b044a285de14dc
Parents: a1f5cc8 f0054a3
Author: Noble Paul <no...@apache.org>
Authored: Fri May 26 09:21:58 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Fri May 26 09:21:58 2017 +0930

----------------------------------------------------------------------
 .../org/apache/solr/cloud/ActionThrottle.java   |  25 +-
 .../org/apache/solr/cloud/ZkController.java     |   2 +
 .../solr/cloud/autoscaling/AutoScaling.java     |  63 ++---
 .../cloud/autoscaling/AutoScalingHandler.java   |   7 +-
 .../cloud/autoscaling/ComputePlanAction.java    |   2 +-
 .../cloud/autoscaling/ExecutePlanAction.java    |   2 +-
 .../solr/cloud/autoscaling/LogPlanAction.java   |   2 +-
 .../cloud/autoscaling/NodeAddedTrigger.java     | 100 ++++----
 .../solr/cloud/autoscaling/NodeLostTrigger.java |  94 ++++----
 .../autoscaling/OverseerTriggerThread.java      |   3 +-
 .../cloud/autoscaling/ScheduledTriggers.java    | 140 +++++++++--
 .../solr/cloud/autoscaling/TriggerAction.java   |   2 +-
 .../solr/cloud/autoscaling/TriggerBase.java     | 107 ++++++++
 .../solr/cloud/autoscaling/TriggerEvent.java    | 153 ++++++++++++
 .../cloud/autoscaling/TriggerEventQueue.java    | 100 ++++++++
 .../src/java/org/apache/solr/util/IdUtils.java  |  39 +++
 .../java/org/apache/solr/util/TimeSource.java   |  57 +++++
 .../apache/solr/cloud/ActionThrottleTest.java   |  21 +-
 .../cloud/autoscaling/NodeAddedTriggerTest.java |  25 +-
 .../cloud/autoscaling/NodeLostTriggerTest.java  |  27 ++-
 .../autoscaling/TriggerIntegrationTest.java     | 241 ++++++++++++++++---
 .../apache/solr/common/cloud/ZkStateReader.java |   2 +
 .../java/org/apache/solr/common/util/Utils.java |  48 +++-
 23 files changed, 1015 insertions(+), 247 deletions(-)
----------------------------------------------------------------------