You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/08/06 04:15:36 UTC
[03/48] lucene-solr:jira/http2: SOLR-12601: Refactor the autoscaling
package to improve readability
SOLR-12601: Refactor the autoscaling package to improve readability
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4602e4de
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4602e4de
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4602e4de
Branch: refs/heads/jira/http2
Commit: 4602e4de458352f8363c3d4078f1df24d2ac9430
Parents: a08eadb
Author: Noble Paul <no...@apache.org>
Authored: Tue Jul 31 13:12:34 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jul 31 13:12:34 2018 +1000
----------------------------------------------------------------------
.../cloud/api/collections/SplitShardCmd.java | 10 +-
.../solr/cloud/autoscaling/TestPolicyCloud.java | 6 +-
.../sim/SimClusterStateProvider.java | 41 +-
.../cloud/autoscaling/sim/TestPolicyCloud.java | 4 +-
.../client/solrj/cloud/autoscaling/Cell.java | 5 +-
.../client/solrj/cloud/autoscaling/Clause.java | 48 +-
.../solrj/cloud/autoscaling/CoresVariable.java | 69 ++
.../cloud/autoscaling/FreeDiskVariable.java | 135 ++++
.../solrj/cloud/autoscaling/NodeVariable.java | 45 ++
.../client/solrj/cloud/autoscaling/Operand.java | 10 +-
.../client/solrj/cloud/autoscaling/Policy.java | 27 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 11 +-
.../solrj/cloud/autoscaling/RangeVal.java | 54 ++
.../cloud/autoscaling/ReplicaVariable.java | 131 ++++
.../client/solrj/cloud/autoscaling/Row.java | 4 +-
.../solrj/cloud/autoscaling/Suggester.java | 2 +-
.../solrj/cloud/autoscaling/Suggestion.java | 740 +------------------
.../client/solrj/cloud/autoscaling/VarType.java | 43 --
.../solrj/cloud/autoscaling/Variable.java | 364 +++++++++
.../solrj/cloud/autoscaling/VariableBase.java | 205 +++++
.../solrj/cloud/autoscaling/Violation.java | 49 ++
.../autoscaling/WithCollectionVarType.java | 160 ----
.../autoscaling/WithCollectionVariable.java | 165 +++++
.../solrj/impl/SolrClientNodeStateProvider.java | 13 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 17 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 2 +-
26 files changed, 1300 insertions(+), 1060 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index ccb111a..b5408f8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -31,10 +31,10 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.client.solrj.cloud.DistributedQueue;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
-import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.overseer.OverseerAction;
@@ -475,7 +475,7 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
Map<String, Object> nodeValues = nodeStateProvider.getNodeValues(parentShardLeader.getNodeName(),
Collections.singletonList(ImplicitSnitch.DISK));
Map<String, Map<String, List<ReplicaInfo>>> infos = nodeStateProvider.getReplicaInfo(parentShardLeader.getNodeName(),
- Collections.singletonList(Suggestion.ConditionType.CORE_IDX.metricsAttribute));
+ Collections.singletonList(Type.CORE_IDX.metricsAttribute));
if (infos.get(collection) == null || infos.get(collection).get(shard) == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing replica information for parent shard leader");
}
@@ -484,11 +484,11 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
Double indexSize = null;
for (ReplicaInfo info : lst) {
if (info.getCore().equals(parentShardLeader.getCoreName())) {
- Number size = (Number)info.getVariable(Suggestion.ConditionType.CORE_IDX.metricsAttribute);
+ Number size = (Number)info.getVariable(Type.CORE_IDX.metricsAttribute);
if (size == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing index size information for parent shard leader");
}
- indexSize = (Double)Suggestion.ConditionType.CORE_IDX.convertVal(size);
+ indexSize = (Double) Type.CORE_IDX.convertVal(size);
break;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
index a25a67a..f1dbad5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
@@ -38,7 +38,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.cloud.autoscaling.Row;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
@@ -120,7 +120,7 @@ public class TestPolicyCloud extends SolrCloudTestCase {
Policy.Session session = config.getPolicy().createSession(cloudManager);
for (Row row : session.getSortedNodes()) {
- Object val = row.getVal(Suggestion.ConditionType.TOTALDISK.tagName, null);
+ Object val = row.getVal(Type.TOTALDISK.tagName, null);
log.info("node: {} , totaldisk : {}, freedisk : {}", row.node, val, row.getVal("freedisk",null));
assertTrue(val != null);
@@ -130,7 +130,7 @@ public class TestPolicyCloud extends SolrCloudTestCase {
for (Row row : session.getSortedNodes()) {
row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> {
for (ReplicaInfo replicaInfo : replicaInfos) {
- if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet();
+ if (replicaInfo.getVariables().containsKey(Type.CORE_IDX.tagName)) count.incrementAndGet();
}
}));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 6024790..abc3ccf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -40,13 +40,14 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -473,10 +474,10 @@ public class SimClusterStateProvider implements ClusterStateProvider {
// mark replica as active
replicaInfo.getVariables().put(ZkStateReader.STATE_PROP, Replica.State.ACTIVE.toString());
// add a property expected in Policy calculations, if missing
- if (replicaInfo.getVariable(Suggestion.ConditionType.CORE_IDX.metricsAttribute) == null) {
- replicaInfo.getVariables().put(Suggestion.ConditionType.CORE_IDX.metricsAttribute, SimCloudManager.DEFAULT_IDX_SIZE_BYTES);
- replicaInfo.getVariables().put(Suggestion.coreidxsize,
- Suggestion.ConditionType.CORE_IDX.convertVal(SimCloudManager.DEFAULT_IDX_SIZE_BYTES));
+ if (replicaInfo.getVariable(Type.CORE_IDX.metricsAttribute) == null) {
+ replicaInfo.getVariables().put(Type.CORE_IDX.metricsAttribute, SimCloudManager.DEFAULT_IDX_SIZE_BYTES);
+ replicaInfo.getVariables().put(Variable.coreidxsize,
+ Type.CORE_IDX.convertVal(SimCloudManager.DEFAULT_IDX_SIZE_BYTES));
}
replicas.add(replicaInfo);
@@ -502,7 +503,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
cloudManager.getMetricManager().registry(registry).counter("UPDATE./update.requests");
cloudManager.getMetricManager().registry(registry).counter("QUERY./select.requests");
cloudManager.getMetricManager().registerGauge(null, registry,
- () -> replicaInfo.getVariable(Suggestion.ConditionType.CORE_IDX.metricsAttribute),
+ () -> replicaInfo.getVariable(Type.CORE_IDX.metricsAttribute),
"", true, "INDEX.sizeInBytes");
if (runLeaderElection) {
simRunLeaderElection(Collections.singleton(replicaInfo.getCollection()), true);
@@ -1085,8 +1086,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
replicaProps.put("SEARCHER.searcher.numDocs", replicasNumDocs);
replicaProps.put("SEARCHER.searcher.maxDoc", replicasNumDocs);
replicaProps.put("SEARCHER.searcher.deletedDocs", 0);
- replicaProps.put(Suggestion.ConditionType.CORE_IDX.metricsAttribute, replicasIndexSize);
- replicaProps.put(Suggestion.coreidxsize, Suggestion.ConditionType.CORE_IDX.convertVal(replicasIndexSize));
+ replicaProps.put(Type.CORE_IDX.metricsAttribute, replicasIndexSize);
+ replicaProps.put(Variable.coreidxsize, Type.CORE_IDX.convertVal(replicasIndexSize));
ReplicaInfo ri = new ReplicaInfo("core_node" + Assign.incAndGetId(stateManager, collectionName, 0),
solrCoreName, collectionName, replicaPosition.shard, replicaPosition.type, subShardNodeName, replicaProps);
@@ -1246,13 +1247,13 @@ public class SimClusterStateProvider implements ClusterStateProvider {
try {
simSetShardValue(collection, s.getName(), "SEARCHER.searcher.deletedDocs", 1, true, false);
simSetShardValue(collection, s.getName(), "SEARCHER.searcher.numDocs", -1, true, false);
- Number indexSize = (Number)ri.getVariable(Suggestion.ConditionType.CORE_IDX.metricsAttribute);
+ Number indexSize = (Number)ri.getVariable(Type.CORE_IDX.metricsAttribute);
if (indexSize != null && indexSize.longValue() > SimCloudManager.DEFAULT_IDX_SIZE_BYTES) {
indexSize = indexSize.longValue() - DEFAULT_DOC_SIZE_BYTES;
- simSetShardValue(collection, s.getName(), Suggestion.ConditionType.CORE_IDX.metricsAttribute,
+ simSetShardValue(collection, s.getName(), Type.CORE_IDX.metricsAttribute,
indexSize.intValue(), false, false);
- simSetShardValue(collection, s.getName(), Suggestion.coreidxsize,
- Suggestion.ConditionType.CORE_IDX.convertVal(indexSize), false, false);
+ simSetShardValue(collection, s.getName(), Variable.coreidxsize,
+ Type.CORE_IDX.convertVal(indexSize), false, false);
} else {
throw new Exception("unexpected indexSize ri=" + ri);
}
@@ -1284,10 +1285,10 @@ public class SimClusterStateProvider implements ClusterStateProvider {
try {
simSetShardValue(collection, s.getName(), "SEARCHER.searcher.deletedDocs", numDocs, false, false);
simSetShardValue(collection, s.getName(), "SEARCHER.searcher.numDocs", 0, false, false);
- simSetShardValue(collection, s.getName(), Suggestion.ConditionType.CORE_IDX.metricsAttribute,
+ simSetShardValue(collection, s.getName(), Type.CORE_IDX.metricsAttribute,
SimCloudManager.DEFAULT_IDX_SIZE_BYTES, false, false);
- simSetShardValue(collection, s.getName(), Suggestion.coreidxsize,
- Suggestion.ConditionType.CORE_IDX.convertVal(SimCloudManager.DEFAULT_IDX_SIZE_BYTES), false, false);
+ simSetShardValue(collection, s.getName(), Variable.coreidxsize,
+ Type.CORE_IDX.convertVal(SimCloudManager.DEFAULT_IDX_SIZE_BYTES), false, false);
} catch (Exception e) {
throw new IOException(e);
}
@@ -1314,13 +1315,13 @@ public class SimClusterStateProvider implements ClusterStateProvider {
simSetShardValue(collection, s.getName(), "SEARCHER.searcher.maxDoc", 1, true, false);
ReplicaInfo ri = getReplicaInfo(leader);
- Number indexSize = (Number)ri.getVariable(Suggestion.ConditionType.CORE_IDX.metricsAttribute);
+ Number indexSize = (Number)ri.getVariable(Type.CORE_IDX.metricsAttribute);
// for each new document increase the size by DEFAULT_DOC_SIZE_BYTES
indexSize = indexSize.longValue() + DEFAULT_DOC_SIZE_BYTES;
- simSetShardValue(collection, s.getName(), Suggestion.ConditionType.CORE_IDX.metricsAttribute,
+ simSetShardValue(collection, s.getName(), Type.CORE_IDX.metricsAttribute,
indexSize.longValue(), false, false);
- simSetShardValue(collection, s.getName(), Suggestion.coreidxsize,
- Suggestion.ConditionType.CORE_IDX.convertVal(indexSize), false, false);
+ simSetShardValue(collection, s.getName(), Variable.coreidxsize,
+ Type.CORE_IDX.convertVal(indexSize), false, false);
} catch (Exception e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
index fad637d..1adbabb 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
@@ -33,7 +33,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.cloud.autoscaling.Row;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.CloudTestUtils;
import org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest;
@@ -95,7 +95,7 @@ public class TestPolicyCloud extends SimSolrCloudTestCase {
for (Row row : session.getSortedNodes()) {
row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> {
for (ReplicaInfo replicaInfo : replicaInfos) {
- if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet();
+ if (replicaInfo.getVariables().containsKey(Type.CORE_IDX.tagName)) count.incrementAndGet();
}
}));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
index 0fa2db2..e222541 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
@@ -20,6 +20,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.util.HashMap;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.Utils;
@@ -28,12 +29,12 @@ import org.apache.solr.common.util.Utils;
*/
public class Cell implements MapWriter {
final int index;
- final Suggestion.ConditionType type;
+ final Type type;
final String name;
Object val, approxVal;
Row row;
- public Cell(int index, String name, Object val, Object approxVal, Suggestion.ConditionType type, Row row) {
+ public Cell(int index, String name, Object val, Object approxVal, Type type, Row row) {
this.index = index;
this.name = name;
this.val = val;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index 8fd815e..5fd9f8b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -31,7 +31,7 @@ import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.util.StrUtils;
@@ -254,7 +254,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
Object expectedVal = null;
ComputedType computedType = null;
Object val = m.get(s);
- ConditionType varType = Suggestion.getTagType(s);
+ Type varType = VariableBase.getTagType(s);
if (varType.meta.isHidden()) {
throw new IllegalArgumentException(formatString("''{0}'' is not allowed in a policy rule : ''{1}'' ", varType.tagName, toJSONString(m)));
}
@@ -309,7 +309,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
}
- private List readListVal(Map m, List val, ConditionType varType, String conditionName) {
+ private List readListVal(Map m, List val, Type varType, String conditionName) {
List list = val;
list = (List) list.stream()
.map(it -> varType.validate(conditionName, it, true))
@@ -354,7 +354,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
public List<Violation> test(Policy.Session session) {
ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
- Suggestion.ViolationCtx ctx = new Suggestion.ViolationCtx(this, session.matrix, computedValueEvaluator);
+ Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator);
if (isPerCollectiontag()) {
Map<String, Map<String, Map<String, ReplicaCount>>> replicaCounts = computeReplicaCounts(session.matrix, computedValueEvaluator);
for (Map.Entry<String, Map<String, Map<String, ReplicaCount>>> e : replicaCounts.entrySet()) {
@@ -448,7 +448,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
return false;
}
- enum ComputedType {
+ public enum ComputedType {
NULL(),
EQUAL() {
@Override
@@ -532,7 +532,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
public static class Condition implements MapWriter {
final String name;
final Object val;
- final ConditionType varType;
+ final Type varType;
final ComputedType computedType;
final Operand op;
private Clause clause;
@@ -541,7 +541,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
this.name = name;
this.val = val;
this.op = op;
- varType = Suggestion.getTagType(name);
+ varType = VariableBase.getTagType(name);
this.computedType = computedType;
this.clause = parent;
}
@@ -668,7 +668,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
*/
public static Object validate(String name, Object val, boolean isRuleVal) {
if (val == null) return null;
- ConditionType info = Suggestion.getTagType(name);
+ Type info = VariableBase.getTagType(name);
if (info == null) throw new RuntimeException("Unknown type :" + name);
return info.validate(name, val, isRuleVal);
}
@@ -723,36 +723,4 @@ public class Clause implements MapWriter, Comparable<Clause> {
public static final String METRICS_PREFIX = "metrics:";
- static class RangeVal implements MapWriter {
- final Number min, max, actual;
-
- RangeVal(Number min, Number max, Number actual) {
- this.min = min;
- this.max = max;
- this.actual = actual;
- }
-
- public boolean match(Number testVal) {
- return Double.compare(testVal.doubleValue(), min.doubleValue()) >= 0 &&
- Double.compare(testVal.doubleValue(), max.doubleValue()) <= 0;
- }
-
- public Double delta(double v) {
-// if (actual != null) return v - actual.doubleValue();
- if (v >= max.doubleValue()) return v - max.doubleValue();
- if (v <= min.doubleValue()) return v - min.doubleValue();
- return 0d;
- }
-
- @Override
- public String toString() {
- return jsonStr();
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put("min", min).put("max", max).putIfNotNull("actual", actual);
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java
new file mode 100644
index 0000000..577717f
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.function.Consumer;
+
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+
+public class CoresVariable extends VariableBase {
+ public CoresVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public Object validate(String name, Object val, boolean isRuleVal) {
+ return VariableBase.getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
+ }
+
+ @Override
+ public void addViolatingReplicas(Violation.Ctx ctx) {
+ for (Row r : ctx.allRows) {
+ if (!ctx.clause.tag.isPass(r)) {
+ r.forEachReplica(replicaInfo -> ctx.currentViolation
+ .addReplica(new Violation.ReplicaInfoAndErr(replicaInfo)
+ .withDelta(ctx.clause.tag.delta(r.getVal(ImplicitSnitch.CORES)))));
+ }
+ }
+
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return;
+ if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
+ for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .hint(Suggester.Hint.SRC_NODE, ctx.violation.node);
+ ctx.addSuggestion(suggester);
+ }
+ }
+ }
+
+ @Override
+ public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> ops, boolean strictMode) {
+ cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() + 1;
+ }
+
+ @Override
+ public void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
+ cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() - 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
new file mode 100644
index 0000000..b598207
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.util.Pair;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORE_IDX;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.TOTALDISK;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+
+public class FreeDiskVariable extends VariableBase {
+
+ public FreeDiskVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public Object convertVal(Object val) {
+ Number value = (Number) super.validate(ImplicitSnitch.DISK, val, false);
+ if (value != null) {
+ value = value.doubleValue() / 1024.0d / 1024.0d / 1024.0d;
+ }
+ return value;
+ }
+
+ @Override
+ public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
+ if (condition.computedType == Clause.ComputedType.PERCENT) {
+ Row r = session.getNode(node);
+ if (r == null) return 0d;
+ return Clause.ComputedType.PERCENT.compute(r.getVal(TOTALDISK.tagName), condition);
+ }
+ throw new IllegalArgumentException("Unsupported type " + condition.computedType);
+ }
+
+
+
+ @Override
+ public int compareViolation(Violation v1, Violation v2) {
+ //TODO use tolerance compare
+ return Double.compare(
+ v1.getViolatingReplicas().stream().mapToDouble(v -> v.delta == null ? 0 : v.delta).max().orElse(0d),
+ v2.getViolatingReplicas().stream().mapToDouble(v3 -> v3.delta == null ? 0 : v3.delta).max().orElse(0d));
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ if (ctx.violation == null) return;
+ if (ctx.violation.replicaCountDelta < 0 && !ctx.violation.getViolatingReplicas().isEmpty()) {
+
+ Comparator<Row> rowComparator = Comparator.comparing(r -> ((Double) r.getVal(ImplicitSnitch.DISK, 0d)));
+ List<Row> matchingNodes = ctx.session.matrix.stream().filter(
+ row -> ctx.violation.getViolatingReplicas()
+ .stream()
+ .anyMatch(p -> row.node.equals(p.replicaInfo.getNode())))
+ .sorted(rowComparator)
+ .collect(Collectors.toList());
+
+
+ for (Row node : matchingNodes) {
+ //lets try to start moving the smallest cores off of the node
+ ArrayList<ReplicaInfo> replicas = new ArrayList<>();
+ node.forEachReplica(replicas::add);
+ replicas.sort((r1, r2) -> {
+ Long s1 = Clause.parseLong(CORE_IDX.tagName, r1.getVariables().get(CORE_IDX.tagName));
+ Long s2 = Clause.parseLong(CORE_IDX.tagName, r2.getVariables().get(CORE_IDX.tagName));
+ if (s1 != null && s2 != null) return s1.compareTo(s2);
+ return 0;
+ });
+ double currentDelta = ctx.violation.getClause().tag.delta(node.getVal(ImplicitSnitch.DISK));
+ for (ReplicaInfo replica : replicas) {
+ if (currentDelta < 1) break;
+ if (replica.getVariables().get(CORE_IDX.tagName) == null) continue;
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .hint(Suggester.Hint.COLL_SHARD, new Pair<>(replica.getCollection(), replica.getShard()))
+ .hint(Suggester.Hint.SRC_NODE, node.node)
+ .forceOperation(true);
+ if (ctx.addSuggestion(suggester) == null) break;
+ currentDelta -= Clause.parseLong(CORE_IDX.tagName, replica.getVariable(CORE_IDX.tagName));
+ }
+ }
+ }
+ }
+
+ //When a replica is added, freedisk should be incremented
+ @Override
+ public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> ops, boolean strictMode) {
+ //go through other replicas of this shard and copy the index size value into this
+ for (Row row : cell.getRow().session.matrix) {
+ row.forEachReplica(replicaInfo -> {
+ if (ri != replicaInfo &&
+ ri.getCollection().equals(replicaInfo.getCollection()) &&
+ ri.getShard().equals(replicaInfo.getShard()) &&
+ ri.getVariable(CORE_IDX.tagName) == null &&
+ replicaInfo.getVariable(CORE_IDX.tagName) != null) {
+ ri.getVariables().put(CORE_IDX.tagName, validate(CORE_IDX.tagName, replicaInfo.getVariable(CORE_IDX.tagName), false));
+ }
+ });
+ }
+ Double idxSize = (Double) validate(CORE_IDX.tagName, ri.getVariable(CORE_IDX.tagName), false);
+ if (idxSize == null) return;
+ Double currFreeDisk = cell.val == null ? 0.0d : (Double) cell.val;
+ cell.val = currFreeDisk - idxSize;
+ }
+
+ @Override
+ public void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
+ Double idxSize = (Double) validate(CORE_IDX.tagName, ri.getVariable(CORE_IDX.tagName), false);
+ if (idxSize == null) return;
+ Double currFreeDisk = cell.val == null ? 0.0d : (Double) cell.val;
+ cell.val = currFreeDisk + idxSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
new file mode 100644
index 0000000..3292800
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import org.apache.solr.common.util.Pair;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+
+public class NodeVariable extends VariableBase {
+ public NodeVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return;
+ if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
+ for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .hint(Suggester.Hint.SRC_NODE, ctx.violation.node)
+ .hint(ctx.violation.shard.equals(ANY) ? Suggester.Hint.COLL : Suggester.Hint.COLL_SHARD,
+ ctx.violation.shard.equals(ANY) ? ctx.violation.coll : new Pair<>(ctx.violation.coll, ctx.violation.shard));
+ ctx.addSuggestion(suggester);
+ }
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
index b458646..3e2368c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
@@ -40,12 +40,12 @@ public enum Operand {
RANGE_EQUAL("", 0) {
@Override
public TestStatus match(Object ruleVal, Object testVal) {
- return ((Clause.RangeVal) ruleVal).match((Number) testVal) ? PASS : FAIL;
+ return ((RangeVal) ruleVal).match((Number) testVal) ? PASS : FAIL;
}
@Override
public Double delta(Object expected, Object actual) {
- return ((Clause.RangeVal) expected).delta(((Number) actual).doubleValue());
+ return ((RangeVal) expected).delta(((Number) actual).doubleValue());
}
@Override
@@ -56,7 +56,7 @@ public enum Operand {
if (hyphenIdx > 0) {
String minS = strVal.substring(0, hyphenIdx).trim();
String maxS = strVal.substring(hyphenIdx + 1, strVal.length()).trim();
- return new Clause.RangeVal(
+ return new RangeVal(
(Number) condition.varType.validate(condition.name, minS, true),
(Number) condition.varType.validate(condition.name, maxS, true),
null
@@ -68,7 +68,7 @@ public enum Operand {
Number num = (Number) condition.varType.validate(condition.name, condition.val, true);
- return new Clause.RangeVal(Math.floor(num.doubleValue()), Math.ceil(num.doubleValue()), num);
+ return new RangeVal(Math.floor(num.doubleValue()), Math.ceil(num.doubleValue()), num);
}
},
EQUAL("", 0) {
@@ -87,7 +87,7 @@ public enum Operand {
RANGE_NOT_EQUAL("", 2) {
@Override
public TestStatus match(Object ruleVal, Object testVal) {
- return ((Clause.RangeVal) ruleVal).match((Number) testVal) ? FAIL : PASS;
+ return ((RangeVal) ruleVal).match((Number) testVal) ? FAIL : PASS;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 879bb74..09aa244 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -38,7 +38,7 @@ import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
@@ -56,8 +56,8 @@ import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.collectingAndThen;
import static java.util.stream.Collectors.toList;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.NODE;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.WITH_COLLECTION;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.NODE;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.WITH_COLLECTION;
/*The class that reads, parses and applies policies specified in
* autoscaling.json
@@ -91,7 +91,7 @@ public class Policy implements MapWriter {
final Map<String, List<Clause>> policies;
final List<Clause> clusterPolicy;
final List<Preference> clusterPreferences;
- final List<Pair<String, ConditionType>> params;
+ final List<Pair<String, Type>> params;
final List<String> perReplicaAttributes;
public Policy() {
@@ -124,7 +124,7 @@ public class Policy implements MapWriter {
.collect(collectingAndThen(toList(), Collections::unmodifiableList));
for (String newParam : new ArrayList<>(newParams)) {
- ConditionType t = Suggestion.getTagType(newParam);
+ Type t = VariableBase.getTagType(newParam);
if(t != null && !t.associatedPerNodeValues.isEmpty()){
for (String s : t.associatedPerNodeValues) {
if(!newParams.contains(s)) newParams.add(s);
@@ -134,8 +134,8 @@ public class Policy implements MapWriter {
this.policies = Collections.unmodifiableMap(
policiesFromMap((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault(POLICIES, emptyMap()), newParams));
- List<Pair<String, Suggestion.ConditionType>> params = newParams.stream()
- .map(s -> new Pair<>(s, Suggestion.getTagType(s)))
+ List<Pair<String, Type>> params = newParams.stream()
+ .map(s -> new Pair<>(s, VariableBase.getTagType(s)))
.collect(toList());
//let this be there always, there is no extra cost
params.add(new Pair<>(WITH_COLLECTION.tagName, WITH_COLLECTION));
@@ -156,7 +156,7 @@ public class Policy implements MapWriter {
this.clusterPreferences = clusterPreferences != null ? Collections.unmodifiableList(clusterPreferences) : DEFAULT_PREFERENCES;
this.params = Collections.unmodifiableList(
buildParams(this.clusterPreferences, this.clusterPolicy, this.policies).stream()
- .map(s -> new Pair<>(s, Suggestion.getTagType(s)))
+ .map(s -> new Pair<>(s, VariableBase.getTagType(s)))
.collect(toList())
);
perReplicaAttributes = readPerReplicaAttrs();
@@ -615,15 +615,4 @@ public class Policy implements MapWriter {
throw new RuntimeException("NO such node found " + node);
}
}
- static final Map<String, Suggestion.ConditionType> validatetypes = new HashMap<>();
- static {
- for (Suggestion.ConditionType t : Suggestion.ConditionType.values())
- validatetypes.put(t.tagName, t);
- }
- public static ConditionType getTagType(String name) {
- ConditionType info = validatetypes.get(name);
- if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = ConditionType.STRING;
- if (info == null && name.startsWith(Clause.METRICS_PREFIX)) info = ConditionType.LAZY;
- return info;
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index ed2a395..f662eff 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -48,7 +48,7 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CoreAdminParams.NODE;
import static org.apache.solr.common.util.Utils.time;
@@ -127,7 +127,7 @@ public class PolicyHelper {
if (replicaInfo != null) {
Object idxSz = replicaInfo.getVariables().get(FREEDISK.perReplicaValue);
if (idxSz != null) {
- diskSpaceReqd.put(shardName, 1.5 * (Double) Suggestion.ConditionType.FREEDISK.validate(null, idxSz, false));
+ diskSpaceReqd.put(shardName, 1.5 * (Double) Variable.Type.FREEDISK.validate(null, idxSz, false));
}
}
}
@@ -213,13 +213,14 @@ public class PolicyHelper {
public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf, SolrCloudManager cloudManager) {
Policy policy = autoScalingConf.getPolicy();
- Suggestion.SuggestionCtx suggestionCtx = new Suggestion.SuggestionCtx();
+ Suggestion.Ctx suggestionCtx = new Suggestion.Ctx();
suggestionCtx.session = policy.createSession(cloudManager);
List<Violation> violations = suggestionCtx.session.getViolations();
for (Violation violation : violations) {
- Suggestion.ConditionType tagType = Suggestion.getTagType(violation.getClause().isPerCollectiontag() ?
+ String name = violation.getClause().isPerCollectiontag() ?
violation.getClause().tag.name :
- violation.getClause().globalTag.name);
+ violation.getClause().globalTag.name;
+ Variable.Type tagType = VariableBase.getTagType(name);
tagType.getSuggestions(suggestionCtx.setViolation(violation));
suggestionCtx.violation = null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
new file mode 100644
index 0000000..16cfe6a
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+
+import org.apache.solr.common.MapWriter;
+
+class RangeVal implements MapWriter {
+ final Number min, max, actual;
+
+ RangeVal(Number min, Number max, Number actual) {
+ this.min = min;
+ this.max = max;
+ this.actual = actual;
+ }
+
+ public boolean match(Number testVal) {
+ return Double.compare(testVal.doubleValue(), min.doubleValue()) >= 0 &&
+ Double.compare(testVal.doubleValue(), max.doubleValue()) <= 0;
+ }
+
+ public Double delta(double v) {
+// if (actual != null) return v - actual.doubleValue();
+ if (v >= max.doubleValue()) return v - max.doubleValue();
+ if (v <= min.doubleValue()) return v - min.doubleValue();
+ return 0d;
+ }
+
+ @Override
+ public String toString() {
+ return jsonStr();
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("min", min).put("max", max).putIfNotNull("actual", actual);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java
new file mode 100644
index 0000000..2f66609
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.solr.common.util.StrUtils;
+
+class ReplicaVariable extends VariableBase {
+
+ public ReplicaVariable(Type type) {
+ super(type);
+ }
+
+ static int getRelevantReplicasCount(Policy.Session session, Clause.Condition cv, String collection, String shard) {
+ AtomicInteger totalReplicasOfInterest = new AtomicInteger(0);
+ Clause clause = cv.getClause();
+ for (Row row : session.matrix) {
+ row.forEachReplica(replicaInfo -> {
+ if (clause.isMatch(replicaInfo, collection, shard))
+ totalReplicasOfInterest.incrementAndGet();
+ });
+ }
+ return totalReplicasOfInterest.get();
+ }
+
+ @Override
+ public Object validate(String name, Object val, boolean isRuleVal) {
+ return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
+ }
+
+
+
+ @Override
+ public Operand getOperand(Operand expected, Object strVal, Clause.ComputedType computedType) {
+ if (computedType == Clause.ComputedType.ALL) return expected;
+ if (strVal instanceof String) {
+ String s = ((String) strVal).trim();
+ int hyphenIdx = s.indexOf('-');
+ if (hyphenIdx > 0) {
+ if (hyphenIdx == s.length() - 1) {
+ throw new IllegalArgumentException("bad range input :" + expected);
+ }
+ if (expected == Operand.EQUAL) return Operand.RANGE_EQUAL;
+ if (expected == Operand.NOT_EQUAL) return Operand.RANGE_NOT_EQUAL;
+ }
+
+ }
+
+ if (expected == Operand.EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) {
+ return Operand.RANGE_EQUAL;
+ }
+ if (expected == Operand.NOT_EQUAL && (computedType != null || !isIntegerEquivalent(strVal)))
+ return Operand.RANGE_NOT_EQUAL;
+
+ return expected;
+ }
+
+ @Override
+ public String postValidate(Clause.Condition condition) {
+ if (condition.computedType == Clause.ComputedType.EQUAL) {
+ if (condition.getClause().tag != null &&
+// condition.getClause().tag.varType == NODE &&
+ (condition.getClause().tag.op == Operand.WILDCARD || condition.getClause().tag.op == Operand.IN)) {
+ return null;
+ } else {
+ return "'replica': '#EQUAL` must be used with 'node':'#ANY'";
+ }
+ }
+ if (condition.computedType == Clause.ComputedType.ALL) {
+ if (condition.getClause().tag != null && (condition.getClause().getTag().op == Operand.IN ||
+ condition.getClause().getTag().op == Operand.WILDCARD)) {
+ return StrUtils.formatString("array value or wild card cannot be used for tag {0} with replica : '#ALL'",
+ condition.getClause().tag.getName());
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard, String node) {
+ if (cv.computedType == Clause.ComputedType.ALL)
+ return Double.valueOf(getRelevantReplicasCount(session, cv, collection, shard));
+ if (cv.computedType == Clause.ComputedType.EQUAL) {
+ int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard);
+ double bucketsCount = getNumBuckets(session, cv.getClause());
+ if (relevantReplicasCount == 0 || bucketsCount == 0) return 0;
+ return (double) relevantReplicasCount / bucketsCount;
+ } else if (cv.computedType == Clause.ComputedType.PERCENT) {
+ return Clause.ComputedType.PERCENT.compute(getRelevantReplicasCount(session, cv, collection, shard), cv);
+ } else {
+ throw new IllegalArgumentException("Unsupported type " + cv.computedType);
+
+ }
+ }
+
+ private int getNumBuckets(Policy.Session session, Clause clause) {
+ if (clause.getTag().getOperand() == Operand.IN) {
+ return ((Collection) clause.getTag().val).size();
+ } else if (clause.getTag().getOperand() == Operand.WILDCARD) {
+ if (clause.getTag().varType == Type.NODE) return session.matrix.size();
+ Set uniqueVals = new HashSet();
+ for (Row matrix : session.matrix) {
+ Object val = matrix.getVal(clause.getTag().name);
+ if (val != null) uniqueVals.add(val);
+ }
+ return uniqueVals.size();
+ } else {
+ throw new IllegalArgumentException("Invalid operand for the tag in " + clause);
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 8a14abd..88e9921 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -54,7 +54,7 @@ public class Row implements MapWriter {
boolean isLive = true;
Policy.Session session;
- public Row(String node, List<Pair<String, Suggestion.ConditionType>> params, List<String> perReplicaAttributes, Policy.Session session) {
+ public Row(String node, List<Pair<String, Variable.Type>> params, List<String> perReplicaAttributes, Policy.Session session) {
this.session = session;
collectionVsShardVsReplicas = session.nodeStateProvider.getReplicaInfo(node, perReplicaAttributes);
if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>();
@@ -64,7 +64,7 @@ public class Row implements MapWriter {
List<String> paramNames = params.stream().map(Pair::first).collect(Collectors.toList());
Map<String, Object> vals = isLive ? session.nodeStateProvider.getNodeValues(node, paramNames) : Collections.emptyMap();
for (int i = 0; i < params.size(); i++) {
- Pair<String, Suggestion.ConditionType> pair = params.get(i);
+ Pair<String, Variable.Type> pair = params.get(i);
cells[i] = new Cell(i, pair.first(), Clause.validate(pair.first(), vals.get(pair.first()), false), null, pair.second(), this);
if (NODE.equals(pair.first())) cells[i].val = node;
if (cells[i].val == null) anyValueMissing = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index 67721ba..9b9b60e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -46,7 +46,7 @@ import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;
/* A suggester is capable of suggesting a collection operation
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index af20fac..3b18e02 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -17,102 +17,18 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.V2RequestSupport;
-import org.apache.solr.client.solrj.cloud.autoscaling.Clause.ComputedType;
import org.apache.solr.client.solrj.cloud.autoscaling.Violation.ReplicaInfoAndErr;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.StrUtils;
-import static java.util.Collections.emptySet;
-import static java.util.Collections.unmodifiableSet;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.parseString;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
public class Suggestion {
- public static final String coreidxsize = "INDEX.sizeInGB";
-
-
- private static final String NULL = "";
-
- @Target(ElementType.FIELD)
- @Retention(RetentionPolicy.RUNTIME)
- @interface Meta {
- String name();
-
- Class type();
-
- String[] associatedPerNodeValue() default NULL;
-
- String associatedPerReplicaValue() default NULL;
-
- String[] enumVals() default NULL;
-
- String[] wildCards() default NULL;
-
- boolean isNodeSpecificVal() default false;
-
- boolean isHidden() default false;
-
- boolean isAdditive() default true;
-
- double min() default -1d;
-
- double max() default -1d;
-
- boolean supportArrayVals() default false;
-
- String metricsKey() default NULL;
-
- Class implementation() default void.class;
-
- ComputedType[] computedValues() default ComputedType.NULL;
- }
-
- public static ConditionType getTagType(String name) {
- return Policy.getTagType(name);
- }
-
- private static Object getOperandAdjustedValue(Object val, Object original) {
- if (original instanceof Clause.Condition) {
- Clause.Condition condition = (Clause.Condition) original;
- if (condition.computedType == null && isIntegerEquivalent(val)) {
- if (condition.op == Operand.LESS_THAN) {
- //replica : '<3'
- val = val instanceof Long ?
- (Long) val - 1 :
- (Double) val - 1;
- } else if (condition.op == Operand.GREATER_THAN) {
- //replica : '>4'
- val = val instanceof Long ?
- (Long) val + 1 :
- (Double) val + 1;
- }
- }
- }
- return val;
- }
-
-
- static class SuggestionCtx {
+ static class Ctx {
public Policy.Session session;
public Violation violation;
private List<Suggester.SuggestionInfo> suggestions = new ArrayList<>();
@@ -128,7 +44,7 @@ public class Suggestion {
}
- public SuggestionCtx setViolation(Violation violation) {
+ public Ctx setViolation(Violation violation) {
this.violation = violation;
return this;
}
@@ -138,658 +54,8 @@ public class Suggestion {
}
}
- static boolean isIntegerEquivalent(Object val) {
- if (val instanceof Number) {
- Number number = (Number) val;
- return Math.ceil(number.doubleValue()) == Math.floor(number.doubleValue());
- } else if (val instanceof String) {
- try {
- double dval = Double.parseDouble((String) val);
- return Math.ceil(dval) == Math.floor(dval);
- } catch (NumberFormatException e) {
- return false;
- }
- } else {
- return false;
- }
-
- }
-
-
- /**
- * Type details of each variable in policies
- */
- public enum ConditionType implements VarType {
- @Meta(name = "withCollection", type = String.class, isNodeSpecificVal = true, implementation = WithCollectionVarType.class)
- WITH_COLLECTION(),
-
- @Meta(name = "collection",
- type = String.class)
- COLL(),
- @Meta(
- name = "shard",
- type = String.class,
- wildCards = {Policy.EACH, Policy.ANY})
- SHARD(),
-
- @Meta(name = "replica",
- type = Double.class,
- min = 0, max = -1,
- computedValues = {ComputedType.EQUAL, ComputedType.PERCENT, ComputedType.ALL})
- REPLICA() {
- @Override
- public Object validate(String name, Object val, boolean isRuleVal) {
- return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
- }
-
- @Override
- public Operand getOperand(Operand expected, Object strVal, ComputedType computedType) {
- if (computedType == ComputedType.ALL) return expected;
- if (strVal instanceof String) {
- String s = ((String) strVal).trim();
- int hyphenIdx = s.indexOf('-');
- if (hyphenIdx > 0) {
- if (hyphenIdx == s.length() - 1) {
- throw new IllegalArgumentException("bad range input :" + expected);
- }
- if (expected == Operand.EQUAL) return Operand.RANGE_EQUAL;
- if (expected == Operand.NOT_EQUAL) return Operand.RANGE_NOT_EQUAL;
- }
-
- }
-
- if (expected == Operand.EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) {
- return Operand.RANGE_EQUAL;
- }
- if (expected == Operand.NOT_EQUAL && (computedType != null || !isIntegerEquivalent(strVal)))
- return Operand.RANGE_NOT_EQUAL;
-
- return expected;
- }
-
- @Override
- public String postValidate(Clause.Condition condition) {
- if (condition.computedType == ComputedType.EQUAL) {
- if (condition.getClause().tag != null &&
-// condition.getClause().tag.varType == NODE &&
- (condition.getClause().tag.op == Operand.WILDCARD || condition.getClause().tag.op == Operand.IN)) {
- return null;
- } else {
- return "'replica': '#EQUAL` must be used with 'node':'#ANY'";
- }
- }
- if (condition.computedType == ComputedType.ALL) {
- if (condition.getClause().tag != null && (condition.getClause().getTag().op == Operand.IN ||
- condition.getClause().getTag().op == Operand.WILDCARD)) {
- return StrUtils.formatString("array value or wild card cannot be used for tag {0} with replica : '#ALL'",
- condition.getClause().tag.getName());
- }
- }
- return null;
- }
-
- @Override
- public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard, String node) {
- if (cv.computedType == ComputedType.ALL)
- return Double.valueOf(getRelevantReplicasCount(session, cv, collection, shard));
- if (cv.computedType == ComputedType.EQUAL) {
- int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard);
- double bucketsCount = getNumBuckets(session, cv.getClause());
- if (relevantReplicasCount == 0 || bucketsCount == 0) return 0;
- return (double) relevantReplicasCount / bucketsCount;
- } else if (cv.computedType == ComputedType.PERCENT) {
- return ComputedType.PERCENT.compute(getRelevantReplicasCount(session, cv, collection, shard), cv);
- } else {
- throw new IllegalArgumentException("Unsupported type " + cv.computedType);
-
- }
- }
-
- private int getNumBuckets(Policy.Session session, Clause clause) {
- if (clause.getTag().getOperand() == Operand.IN) {
- return ((Collection) clause.getTag().val).size();
- } else if (clause.getTag().getOperand() == Operand.WILDCARD) {
- if (clause.getTag().varType == NODE) return session.matrix.size();
- Set uniqueVals = new HashSet();
- for (Row matrix : session.matrix) {
- Object val = matrix.getVal(clause.getTag().name);
- if (val != null) uniqueVals.add(val);
- }
- return uniqueVals.size();
- } else {
- throw new IllegalArgumentException("Invalid operand for the tag in " + clause);
- }
-
- }
- },
- @Meta(name = ImplicitSnitch.PORT,
- type = Long.class,
- min = 1,
- max = 65535,
- supportArrayVals = true,
- wildCards = Policy.EACH
- )
- PORT(),
- @Meta(name = "ip_1",
- type = Long.class,
- min = 0,
- max = 255,
- supportArrayVals = true,
- wildCards = Policy.EACH)
- IP_1(),
- @Meta(name = "ip_2",
- type = Long.class,
- min = 0,
- max = 255,
- supportArrayVals = true,
- wildCards = Policy.EACH)
- IP_2(),
- @Meta(name = "ip_3",
- type = Long.class,
- min = 0,
- max = 255,
- supportArrayVals = true,
- wildCards = Policy.EACH)
- IP_3(),
- @Meta(name = "ip_4",
- type = Long.class,
- min = 0,
- max = 255,
- supportArrayVals = true,
- wildCards = Policy.EACH)
- IP_4(),
- @Meta(name = ImplicitSnitch.DISK,
- type = Double.class,
- min = 0,
- isNodeSpecificVal = true,
- associatedPerReplicaValue = coreidxsize,
- associatedPerNodeValue = "totaldisk",
- computedValues = ComputedType.PERCENT)
- FREEDISK() {
- @Override
- public Object convertVal(Object val) {
- Number value = (Number) super.validate(ImplicitSnitch.DISK, val, false);
- if (value != null) {
- value = value.doubleValue() / 1024.0d / 1024.0d / 1024.0d;
- }
- return value;
- }
-
- @Override
- public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
- if (condition.computedType == ComputedType.PERCENT) {
- Row r = session.getNode(node);
- if (r == null) return 0d;
- return ComputedType.PERCENT.compute(r.getVal(TOTALDISK.tagName), condition);
- }
- throw new IllegalArgumentException("Unsupported type " + condition.computedType);
- }
-
-
-
- @Override
- public int compareViolation(Violation v1, Violation v2) {
- //TODO use tolerance compare
- return Double.compare(
- v1.getViolatingReplicas().stream().mapToDouble(v -> v.delta == null ? 0 : v.delta).max().orElse(0d),
- v2.getViolatingReplicas().stream().mapToDouble(v3 -> v3.delta == null ? 0 : v3.delta).max().orElse(0d));
- }
-
- @Override
- public void getSuggestions(SuggestionCtx ctx) {
- if (ctx.violation == null) return;
- if (ctx.violation.replicaCountDelta < 0 && !ctx.violation.getViolatingReplicas().isEmpty()) {
-
- Comparator<Row> rowComparator = Comparator.comparing(r -> ((Double) r.getVal(ImplicitSnitch.DISK, 0d)));
- List<Row> matchingNodes = ctx.session.matrix.stream().filter(
- row -> ctx.violation.getViolatingReplicas()
- .stream()
- .anyMatch(p -> row.node.equals(p.replicaInfo.getNode())))
- .sorted(rowComparator)
- .collect(Collectors.toList());
-
-
- for (Row node : matchingNodes) {
- //lets try to start moving the smallest cores off of the node
- ArrayList<ReplicaInfo> replicas = new ArrayList<>();
- node.forEachReplica(replicas::add);
- replicas.sort((r1, r2) -> {
- Long s1 = Clause.parseLong(ConditionType.CORE_IDX.tagName, r1.getVariables().get(ConditionType.CORE_IDX.tagName));
- Long s2 = Clause.parseLong(ConditionType.CORE_IDX.tagName, r2.getVariables().get(ConditionType.CORE_IDX.tagName));
- if (s1 != null && s2 != null) return s1.compareTo(s2);
- return 0;
- });
- double currentDelta = ctx.violation.getClause().tag.delta(node.getVal(ImplicitSnitch.DISK));
- for (ReplicaInfo replica : replicas) {
- if (currentDelta < 1) break;
- if (replica.getVariables().get(ConditionType.CORE_IDX.tagName) == null) continue;
- Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
- .hint(Suggester.Hint.COLL_SHARD, new Pair<>(replica.getCollection(), replica.getShard()))
- .hint(Suggester.Hint.SRC_NODE, node.node)
- .forceOperation(true);
- if (ctx.addSuggestion(suggester) == null) break;
- currentDelta -= Clause.parseLong(ConditionType.CORE_IDX.tagName, replica.getVariable(ConditionType.CORE_IDX.tagName));
- }
- }
- }
- }
-
- //When a replica is added, freedisk should be incremented
- @Override
- public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> ops, boolean strictMode) {
- //go through other replicas of this shard and copy the index size value into this
- for (Row row : cell.getRow().session.matrix) {
- row.forEachReplica(replicaInfo -> {
- if (ri != replicaInfo &&
- ri.getCollection().equals(replicaInfo.getCollection()) &&
- ri.getShard().equals(replicaInfo.getShard()) &&
- ri.getVariable(CORE_IDX.tagName) == null &&
- replicaInfo.getVariable(CORE_IDX.tagName) != null) {
- ri.getVariables().put(CORE_IDX.tagName, validate(CORE_IDX.tagName, replicaInfo.getVariable(CORE_IDX.tagName), false));
- }
- });
- }
- Double idxSize = (Double) validate(CORE_IDX.tagName, ri.getVariable(CORE_IDX.tagName), false);
- if (idxSize == null) return;
- Double currFreeDisk = cell.val == null ? 0.0d : (Double) cell.val;
- cell.val = currFreeDisk - idxSize;
- }
-
- @Override
- public void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
- Double idxSize = (Double) validate(CORE_IDX.tagName, ri.getVariable(CORE_IDX.tagName), false);
- if (idxSize == null) return;
- Double currFreeDisk = cell.val == null ? 0.0d : (Double) cell.val;
- cell.val = currFreeDisk + idxSize;
- }
- },
-
- @Meta(name = "totaldisk",
- type = Double.class,
- isHidden = true)
- TOTALDISK() {
- @Override
- public Object convertVal(Object val) {
- return FREEDISK.convertVal(val);
- }
- },
-
- @Meta(name = coreidxsize,
- type = Double.class,
- isNodeSpecificVal = true,
- isHidden = true,
- min = 0,
- metricsKey = "INDEX.sizeInBytes")
- CORE_IDX() {
- @Override
- public Object convertVal(Object val) {
- return FREEDISK.convertVal(val);
- }
- },
- @Meta(name = ImplicitSnitch.NODEROLE,
- type = String.class,
- enumVals = "overseer")
- NODE_ROLE(),
-
- @Meta(name = ImplicitSnitch.CORES,
- type = Long.class,
- min = 0)
- CORES() {
- @Override
- public Object validate(String name, Object val, boolean isRuleVal) {
- return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
- }
-
- @Override
- public void addViolatingReplicas(ViolationCtx ctx) {
- for (Row r : ctx.allRows) {
- if (!ctx.clause.tag.isPass(r)) {
- r.forEachReplica(replicaInfo -> ctx.currentViolation
- .addReplica(new ReplicaInfoAndErr(replicaInfo)
- .withDelta(ctx.clause.tag.delta(r.getVal(ImplicitSnitch.CORES)))));
- }
- }
-
- }
-
- @Override
- public void getSuggestions(SuggestionCtx ctx) {
- if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return;
- if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
- for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
- Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
- .hint(Suggester.Hint.SRC_NODE, ctx.violation.node);
- ctx.addSuggestion(suggester);
- }
- }
- }
-
- @Override
- public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> ops, boolean strictMode) {
- cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() + 1;
- }
-
- @Override
- public void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
- cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() - 1;
- }
- },
-
- @Meta(name = ImplicitSnitch.SYSLOADAVG,
- type = Double.class,
- min = 0,
- max = 100,
- isNodeSpecificVal = true)
- SYSLOADAVG(),
-
- @Meta(name = ImplicitSnitch.HEAPUSAGE,
- type = Double.class,
- min = 0,
- isNodeSpecificVal = true)
- HEAPUSAGE(),
- @Meta(name = "NUMBER",
- type = Long.class,
- min = 0)
- NUMBER(),
-
- @Meta(name = "STRING",
- type = String.class,
- wildCards = Policy.EACH,
- supportArrayVals = true)
- STRING(),
-
- @Meta(name = "node",
- type = String.class,
- isNodeSpecificVal = true,
- wildCards = {Policy.ANY, Policy.EACH},
- supportArrayVals = true)
- NODE() {
- @Override
- public void getSuggestions(SuggestionCtx ctx) {
- if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return;
- if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
- for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
- Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
- .hint(Suggester.Hint.SRC_NODE, ctx.violation.node)
- .hint(ctx.violation.shard.equals(ANY) ? Suggester.Hint.COLL : Suggester.Hint.COLL_SHARD,
- ctx.violation.shard.equals(ANY) ? ctx.violation.coll : new Pair<>(ctx.violation.coll, ctx.violation.shard));
- ctx.addSuggestion(suggester);
- }
- }
-
- }
-
- },
-
- @Meta(name = "LAZY",
- type = void.class)
- LAZY() {
- @Override
- public Object validate(String name, Object val, boolean isRuleVal) {
- return parseString(val);
- }
-
- @Override
- public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
- return op.match(parseString(val), parseString(inputVal)) == Clause.TestStatus.PASS;
- }
-
- @Override
- public void getSuggestions(SuggestionCtx ctx) {
- perNodeSuggestions(ctx);
- }
- },
-
- @Meta(name = ImplicitSnitch.DISKTYPE,
- type = String.class,
- enumVals = {"ssd", "rotational"},
- supportArrayVals = true)
- DISKTYPE() {
- @Override
- public void getSuggestions(SuggestionCtx ctx) {
- perNodeSuggestions(ctx);
- }
-
-
- };
-
- public final String tagName;
- public final Class type;
- public Meta meta;
-
- public final Set<String> vals;
- public final Number min;
- public final Number max;
- public final Boolean additive;
- public final Set<String> wildCards;
- public final String perReplicaValue;
- public final Set<String> associatedPerNodeValues;
- public final String metricsAttribute;
- public final Set<ComputedType> supportedComputedTypes;
- private final VarType impl;
-
-
- ConditionType() {
- try {
- meta = ConditionType.class.getField(name()).getAnnotation(Meta.class);
- if (meta == null) {
- throw new RuntimeException("Invalid type, should have a @Meta annotation " + name());
- }
- } catch (NoSuchFieldException e) {
- //cannot happen
- }
- if (meta.implementation() != void.class) {
- try {
- impl = (VarType) meta.implementation().newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Unable to instantiate: " + meta.implementation().getName());
- }
- } else {
- impl = null;
- }
- this.tagName = meta.name();
- this.type = meta.type();
-
- this.vals = readSet(meta.enumVals());
- this.max = readNum(meta.max());
- this.min = readNum(meta.min());
- this.perReplicaValue = readStr(meta.associatedPerReplicaValue());
- this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue());
- this.additive = meta.isAdditive();
- this.metricsAttribute = readStr(meta.metricsKey());
- this.supportedComputedTypes = meta.computedValues()[0] == ComputedType.NULL ?
- emptySet() :
- unmodifiableSet(new HashSet(Arrays.asList(meta.computedValues())));
- this.wildCards = readSet(meta.wildCards());
-
- }
-
- public String getTagName() {
- return meta.name();
- }
-
- private String readStr(String s) {
- return NULL.equals(s) ? null : s;
- }
-
- private Number readNum(double v) {
- return v == -1 ? null :
- (Number) validate(null, v, true);
- }
-
- Set<String> readSet(String[] vals) {
- if (NULL.equals(vals[0])) return emptySet();
- return unmodifiableSet(new HashSet<>(Arrays.asList(vals)));
- }
-
- @Override
- public void getSuggestions(SuggestionCtx ctx) {
- if (impl != null) {
- impl.getSuggestions(ctx);
- return;
- }
- perNodeSuggestions(ctx);
- }
-
- @Override
- public void addViolatingReplicas(ViolationCtx ctx) {
- if (impl != null) {
- impl.addViolatingReplicas(ctx);
- return;
- }
- for (Row row : ctx.allRows) {
- if (ctx.clause.tag.varType.meta.isNodeSpecificVal() && !row.node.equals(ctx.tagKey)) continue;
- collectViolatingReplicas(ctx, row);
- }
- }
-
- public Operand getOperand(Operand expected, Object val, ComputedType computedType) {
- return expected;
- }
-
-
- public Object convertVal(Object val) {
- return val;
- }
-
- public String postValidate(Clause.Condition condition) {
- return null;
- }
-
- public Object validate(String name, Object val, boolean isRuleVal) {
- if (val instanceof Clause.Condition) {
- Clause.Condition condition = (Clause.Condition) val;
- val = condition.op.readRuleValue(condition);
- if (val != condition.val) return val;
- }
- if (name == null) name = this.tagName;
- if (type == Double.class) {
- Double num = Clause.parseDouble(name, val);
- if (isRuleVal) {
- if (min != null)
- if (Double.compare(num, min.doubleValue()) == -1)
- throw new RuntimeException(name + ": " + val + " must be greater than " + min);
- if (max != null)
- if (Double.compare(num, max.doubleValue()) == 1)
- throw new RuntimeException(name + ": " + val + " must be less than " + max);
- }
- return num;
- } else if (type == Long.class) {
- Long num = Clause.parseLong(name, val);
- if (isRuleVal) {
- if (min != null)
- if (num < min.longValue())
- throw new RuntimeException(name + ": " + val + " must be greater than " + min);
- if (max != null)
- if (num > max.longValue())
- throw new RuntimeException(name + ": " + val + " must be less than " + max);
- }
- return num;
- } else if (type == String.class) {
- if (isRuleVal && !vals.isEmpty() && !vals.contains(val))
- throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(vals, ','));
- return val;
- } else {
- throw new RuntimeException("Invalid type ");
- }
-
- }
-
- /**
- * Simulate a replica addition to a node in the cluster
- */
- public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode) {
- if (impl != null) impl.projectAddReplica(cell, ri, opCollector, strictMode);
- }
-
- public void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
- if (impl != null) {
- impl.projectRemoveReplica(cell, ri, opCollector);
- }
- }
-
- @Override
- public int compareViolation(Violation v1, Violation v2) {
- if (impl != null) return impl.compareViolation(v1, v2);
- if (v2.replicaCountDelta == null || v1.replicaCountDelta == null) return 0;
- if (Math.abs(v1.replicaCountDelta) == Math.abs(v2.replicaCountDelta)) return 0;
- return Math.abs(v1.replicaCountDelta) < Math.abs(v2.replicaCountDelta) ? -1 : 1;
- }
-
- @Override
- public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
- if (impl != null) return impl.computeValue(session, condition, collection, shard, node);
- return condition.val;
- }
-
- @Override
- public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
- if (impl != null) return impl.match(inputVal, op, val, name, row);
- return op.match(val, validate(name, inputVal, false)) == Clause.TestStatus.PASS;
- }
- }
-
- private static void collectViolatingReplicas(ViolationCtx ctx, Row row) {
- if (ctx.clause.tag.varType.meta.isNodeSpecificVal()) {
- row.forEachReplica(replica -> {
- if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard())) {
- ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica)
- .withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
- }
- });
- } else {
- row.forEachReplica(replica -> {
- if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
- if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
- if(!ctx.currentViolation.getClause().matchShard(replica.getShard(), ctx.currentViolation.shard)) return;
- if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
- return;
- ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
- });
-
- }
-
-
- }
-
- private static int getRelevantReplicasCount(Policy.Session session, Clause.Condition cv, String collection, String shard) {
- AtomicInteger totalReplicasOfInterest = new AtomicInteger(0);
- Clause clause = cv.getClause();
- for (Row row : session.matrix) {
- row.forEachReplica(replicaInfo -> {
- if (clause.isMatch(replicaInfo, collection, shard))
- totalReplicasOfInterest.incrementAndGet();
- });
- }
- return totalReplicasOfInterest.get();
- }
-
- static class ViolationCtx {
- final Function<Clause.Condition, Object> evaluator;
- String tagKey;
- Clause clause;
- ReplicaCount count;
- Violation currentViolation;
- List<Row> allRows;
- List<Violation> allViolations = new ArrayList<>();
-
- public ViolationCtx(Clause clause, List<Row> allRows, Function<Clause.Condition, Object> evaluator) {
- this.allRows = allRows;
- this.clause = clause;
- this.evaluator = evaluator;
- }
-
- public ViolationCtx reset(String tagKey, ReplicaCount count, Violation currentViolation) {
- this.tagKey = tagKey;
- this.count = count;
- this.currentViolation = currentViolation;
- allViolations.add(currentViolation);
- this.clause = currentViolation.getClause();
- return this;
- }
- }
- private static void perNodeSuggestions(SuggestionCtx ctx) {
+ static void perNodeSuggestions(Ctx ctx) {
if (ctx.violation == null) return;
for (ReplicaInfoAndErr e : ctx.violation.getViolatingReplicas()) {
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VarType.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VarType.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VarType.java
deleted file mode 100644
index 00224a9..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VarType.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.cloud.autoscaling;
-
-import java.util.function.Consumer;
-
-/**
- * A Variable Type used in Autoscaling policy rules
- */
-public interface VarType {
- boolean match(Object inputVal, Operand op, Object val, String name, Row row);
-
- void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode);
-
- void addViolatingReplicas(Suggestion.ViolationCtx ctx);
-
- default void getSuggestions(Suggestion.SuggestionCtx ctx) {
- }
-
- default Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
- return condition.val;
- }
-
- int compareViolation(Violation v1, Violation v2);
-
- default void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
- }
-}