You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2018/07/17 21:13:03 UTC
[28/48] lucene-solr:jira/lucene-2562: SOLR-11986: Allow percentage in
freedisk attribute in autoscaling policy rules
SOLR-11986: Allow percentage in freedisk attribute in autoscaling policy rules
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/738dadca
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/738dadca
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/738dadca
Branch: refs/heads/jira/lucene-2562
Commit: 738dadca65dd61cc0e428dde0644821ebefd0cf0
Parents: 3cb436d
Author: Noble Paul <no...@apache.org>
Authored: Mon Jul 16 17:36:02 2018 +1000
Committer: Steve Rowe <sa...@apache.org>
Committed: Tue Jul 17 12:22:30 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solr/cloud/autoscaling/TestPolicyCloud.java | 9 +-
.../cloud/autoscaling/sim/TestPolicyCloud.java | 2 +-
.../client/solrj/cloud/autoscaling/Clause.java | 101 ++++--
.../client/solrj/cloud/autoscaling/Policy.java | 16 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 2 +-
.../client/solrj/cloud/autoscaling/Row.java | 2 +-
.../solrj/cloud/autoscaling/Suggestion.java | 330 ++++++++++++++-----
.../solrj/cloud/autoscaling/Violation.java | 2 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 33 +-
.../java/org/apache/solr/common/MapWriter.java | 5 +
.../solrj/cloud/autoscaling/TestPolicy.java | 117 +++++--
12 files changed, 468 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b7df7b6..d8823db 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -110,6 +110,8 @@ New Features
* SOLR-12441: New NestedUpdateProcessorFactory (URP) to populate special fields _nest_parent_ and _nest_path_ of nested
(child) documents. It will generate a uniqueKey of nested docs if they were blank too. (Moshe Bla, David Smiley)
+* SOLR-11986: Allow percentage in freedisk attribute in autoscaling policy rules (noble)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
index 19324e7..a25a67a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
@@ -119,8 +119,15 @@ public class TestPolicyCloud extends SolrCloudTestCase {
Policy.Session session = config.getPolicy().createSession(cloudManager);
+ for (Row row : session.getSortedNodes()) {
+ Object val = row.getVal(Suggestion.ConditionType.TOTALDISK.tagName, null);
+ log.info("node: {} , totaldisk : {}, freedisk : {}", row.node, val, row.getVal("freedisk",null));
+ assertTrue(val != null);
+
+ }
+
count .set(0);
- for (Row row : session.getSorted()) {
+ for (Row row : session.getSortedNodes()) {
row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> {
for (ReplicaInfo replicaInfo : replicaInfos) {
if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
index b6368d6..fad637d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestPolicyCloud.java
@@ -92,7 +92,7 @@ public class TestPolicyCloud extends SimSolrCloudTestCase {
Policy.Session session = config.getPolicy().createSession(cluster);
AtomicInteger count = new AtomicInteger(0);
- for (Row row : session.getSorted()) {
+ for (Row row : session.getSortedNodes()) {
row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> {
for (ReplicaInfo replicaInfo : replicaInfos) {
if (replicaInfo.getVariables().containsKey(Suggestion.ConditionType.CORE_IDX.tagName)) count.incrementAndGet();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index 60ebf9b..06dc7a0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -30,6 +30,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
+import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.util.StrUtils;
@@ -116,6 +117,10 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
}
+ public static Clause create(String json) {
+ return create((Map<String, Object>) Utils.fromJSONString(json));
+ }
+
public static Clause create(Map<String, Object> m) {
Clause clause = new Clause(m);
return clause.hasComputedValue() ?
@@ -150,7 +155,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
private Condition evaluateValue(Condition condition, Function<Condition, Object> computedValueEvaluator) {
if (condition == null) return null;
- if (condition.computationType == null) return condition;
+ if (condition.computedType == null) return condition;
Object val = computedValueEvaluator.apply(condition);
val = condition.op.readRuleValue(new Condition(condition.name, val, condition.op, null, null));
return new Condition(condition.name, val, condition.op, null, this);
@@ -182,9 +187,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
private boolean hasComputedValue() {
- if (replica != null && replica.computationType != null) return true;
- if (tag != null && tag.computationType != null) return true;
- if (globalTag != null && globalTag.computationType != null) return true;
+ if (replica != null && replica.computedType != null) return true;
+ if (tag != null && tag.computedType != null) return true;
+ if (globalTag != null && globalTag.computedType != null) return true;
return false;
}
@@ -233,9 +238,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
Condition parse(String s, Map m) {
Object expectedVal = null;
- ComputationType computationType = null;
+ ComputedType computedType = null;
Object val = m.get(s);
- Suggestion.ConditionType varType = Suggestion.getTagType(s);
+ ConditionType varType = Suggestion.getTagType(s);
+ if (varType.isHidden) {
+ throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed in a policy rule : ''{1}'' ", varType.tagName, Utils.toJSONString(m)));
+ }
try {
String conditionName = s.trim();
Operand operand = null;
@@ -244,32 +252,39 @@ public class Clause implements MapWriter, Comparable<Clause> {
expectedVal = Policy.ANY;
} else if (val instanceof String) {
String strVal = ((String) val).trim();
+ val = strVal;
if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
else operand = Operand.EQUAL;
strVal = strVal.substring(Operand.EQUAL == operand || WILDCARD == operand ? 0 : 1);
- for (ComputationType t : ComputationType.values()) {
+ for (ComputedType t : ComputedType.values()) {
String changedVal = t.match(strVal);
if (changedVal != null) {
- computationType = t;
+ computedType = t;
strVal = changedVal;
- if (varType == null || !varType.supportComputed(computationType, this)) {
- throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed for variable : ''{1}'' , in condition : ''{2}'' ",
+ if (varType == null || !varType.supportedComputedTypes.contains(computedType)) {
+ throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed for variable : ''{1}'' , in clause : ''{2}'' ",
t, conditionName, Utils.toJSONString(m)));
}
}
}
- operand = varType == null ? operand : varType.getOperand(operand, strVal, computationType);
- expectedVal = validate(s, new Condition(s, strVal, operand, computationType, null), true);
+ if (computedType == null && ((String) val).charAt(0) == '#' && !varType.wildCards.contains(val)) {
+ throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not an allowed value for ''{1}'' , in clause : ''{2}'' . Supported value is : {3}",
+ val, conditionName, Utils.toJSONString(m), varType.wildCards));
+
+
+ }
+ operand = varType == null ? operand : varType.getOperand(operand, strVal, computedType);
+ expectedVal = validate(s, new Condition(s, strVal, operand, computedType, null), true);
} else if (val instanceof Number) {
operand = Operand.EQUAL;
operand = varType.getOperand(operand, val, null);
expectedVal = validate(s, new Condition(s, val, operand, null, null), true);
}
- return new Condition(conditionName, expectedVal, operand, computationType, this);
+ return new Condition(conditionName, expectedVal, operand, computedType, this);
} catch (IllegalArgumentException iae) {
throw iae;
@@ -290,16 +305,17 @@ public class Clause implements MapWriter, Comparable<Clause> {
computedValueEvaluator.shardName = shardVsCount.getKey();
if (!shard.isPass(computedValueEvaluator.shardName)) continue;
for (Map.Entry<String, ReplicaCount> counts : shardVsCount.getValue().entrySet()) {
+ if(tag.varType.isPerNodeValue) computedValueEvaluator.node = counts.getKey();
SealedClause sealedClause = getSealedClause(computedValueEvaluator);
ReplicaCount replicas = counts.getValue();
if (!sealedClause.replica.isPass(replicas)) {
Violation violation = new Violation(sealedClause,
computedValueEvaluator.collName,
computedValueEvaluator.shardName,
- tag.name.equals("node") ? counts.getKey() : null,
+ tag.varType.isPerNodeValue ? computedValueEvaluator.node : null,
counts.getValue(),
sealedClause.getReplica().delta(replicas),
- counts.getKey());
+ tag.varType.isPerNodeValue? null: counts.getKey());
tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
}
}
@@ -309,7 +325,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
for (Row r : session.matrix) {
SealedClause sealedClause = getSealedClause(computedValueEvaluator);
if (!sealedClause.getGlobalTag().isPass(r)) {
- Suggestion.ConditionType.CORES.addViolatingReplicas(ctx.reset(null, null,
+ ConditionType.CORES.addViolatingReplicas(ctx.reset(null, null,
new Violation(sealedClause, null, null, r.node, r.getVal(sealedClause.globalTag.name), sealedClause.globalTag.delta(r.getVal(globalTag.name)), null)));
}
}
@@ -322,6 +338,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
ComputedValueEvaluator computedValueEvaluator) {
Map<String, Map<String, Map<String, ReplicaCount>>> collVsShardVsTagVsCount = new HashMap<>();
for (Row row : allRows) {
+ computedValueEvaluator.node = row.node;
for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
String collectionName = colls.getKey();
if (!collection.isPass(collectionName)) continue;
@@ -335,10 +352,19 @@ public class Clause implements MapWriter, Comparable<Clause> {
computedValueEvaluator.collName = collectionName;
computedValueEvaluator.shardName = shardName;
SealedClause sealedClause = getSealedClause(computedValueEvaluator);
- boolean pass = sealedClause.getTag().isPass(tagVal);
- tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
- if (pass) {
- tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
+ Condition t = sealedClause.getTag();
+ if(t.varType.isPerNodeValue){
+ boolean pass = t.getOperand().match(t.val, tagVal) == TestStatus.PASS;
+ tagVsCount.computeIfAbsent(row.node, s -> new ReplicaCount());
+ if(pass) {
+ tagVsCount.get(row.node).increment(shards.getValue());
+ }
+ } else {
+ boolean pass = sealedClause.getTag().isPass(tagVal);
+ tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
+ if (pass) {
+ tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
+ }
}
}
}
@@ -346,7 +372,8 @@ public class Clause implements MapWriter, Comparable<Clause> {
return collVsShardVsTagVsCount;
}
- enum ComputationType {
+ enum ComputedType {
+ NULL(),
EQUAL() {
@Override
public String wrap(String value) {
@@ -388,6 +415,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
@Override
+ public Object compute(Object val, Condition c) {
+ if (val == null || Clause.parseDouble(c.name, val) == 0) return 0d;
+ return Clause.parseDouble(c.name, val) * Clause.parseDouble(c.name, c.val).doubleValue() / 100;
+ }
+
+ @Override
public String toString() {
return "%";
}
@@ -402,29 +435,34 @@ public class Clause implements MapWriter, Comparable<Clause> {
public String wrap(String value) {
return value;
}
+
+ public Object compute(Object val, Condition c) {
+ return val;
+ }
+
}
public static class Condition implements MapWriter {
final String name;
final Object val;
- final Suggestion.ConditionType varType;
- final ComputationType computationType;
+ final ConditionType varType;
+ final ComputedType computedType;
final Operand op;
private Clause clause;
- Condition(String name, Object val, Operand op, ComputationType computationType, Clause parent) {
+ Condition(String name, Object val, Operand op, ComputedType computedType, Clause parent) {
this.name = name;
this.val = val;
this.op = op;
varType = Suggestion.getTagType(name);
- this.computationType = computationType;
+ this.computedType = computedType;
this.clause = parent;
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
String value = op.wrap(val);
- if (computationType != null) value = computationType.wrap(value);
+ if (computedType != null) value = computedType.wrap(value);
ew.put(name, value);
}
@@ -438,12 +476,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
boolean isPass(Object inputVal) {
- if (computationType != null) {
+ if (computedType != null) {
throw new IllegalStateException("This is supposed to be called only from a Condition with no computed value or a SealedCondition");
}
if (inputVal instanceof ReplicaCount) inputVal = ((ReplicaCount) inputVal).getVal(getClause().type);
- if (varType == Suggestion.ConditionType.LAZY) { // we don't know the type
+ if (varType == ConditionType.LAZY) { // we don't know the type
return op.match(parseString(val), parseString(inputVal)) == PASS;
} else {
return op.match(val, validate(name, inputVal, false)) == PASS;
@@ -514,14 +552,15 @@ public class Clause implements MapWriter, Comparable<Clause> {
final Policy.Session session;
String collName = null;
String shardName = null;
+ String node = null;
public ComputedValueEvaluator(Policy.Session session) {
this.session = session;
}
@Override
- public Object apply(Condition computedValue) {
- return computedValue.varType.computeValue(session, computedValue, collName, shardName);
+ public Object apply(Condition computedCondition) {
+ return computedCondition.varType.computeValue(session, computedCondition, collName, shardName, node);
}
}
@@ -534,7 +573,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
*/
public static Object validate(String name, Object val, boolean isRuleVal) {
if (val == null) return null;
- Suggestion.ConditionType info = Suggestion.getTagType(name);
+ ConditionType info = Suggestion.getTagType(name);
if (info == null) throw new RuntimeException("Unknown type :" + name);
return info.validate(name, val, isRuleVal);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index f7b8c03..9addc67 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -38,6 +38,7 @@ import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
@@ -87,7 +88,7 @@ public class Policy implements MapWriter {
final Map<String, List<Clause>> policies;
final List<Clause> clusterPolicy;
final List<Preference> clusterPreferences;
- final List<Pair<String, Suggestion.ConditionType>> params;
+ final List<Pair<String, ConditionType>> params;
final List<String> perReplicaAttributes;
public Policy() {
@@ -119,6 +120,15 @@ public class Policy implements MapWriter {
})
.collect(collectingAndThen(toList(), Collections::unmodifiableList));
+ for (String newParam : new ArrayList<>(newParams)) {
+ ConditionType t = Suggestion.getTagType(newParam);
+ if(t != null && !t.associatedPerNodeValues.isEmpty()){
+ for (String s : t.associatedPerNodeValues) {
+ if(!newParams.contains(s)) newParams.add(s);
+ }
+ }
+ }
+
this.policies = Collections.unmodifiableMap(
policiesFromMap((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault(POLICIES, emptyMap()), newParams));
this.params = Collections.unmodifiableList(newParams.stream()
@@ -129,7 +139,7 @@ public class Policy implements MapWriter {
private List<String> readPerReplicaAttrs() {
return this.params.stream()
- .map(s -> Suggestion.tagVsPerReplicaVal.get(s.first()))
+ .map(s -> s.second().perReplicaValue)
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
@@ -571,7 +581,7 @@ public class Policy implements MapWriter {
return Utils.toJSONString(toMap(new LinkedHashMap<>()));
}
- public List<Row> getSorted() {
+ public List<Row> getSortedNodes() {
return Collections.unmodifiableList(matrix);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 7785fda..a965bf8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -182,7 +182,7 @@ public class PolicyHelper {
public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) {
Policy.Session session = policy.createSession(cloudManager);
- List<Row> sorted = session.getSorted();
+ List<Row> sorted = session.getSortedNodes();
List<Violation> violations = session.getViolations();
List<Preference> clusterPreferences = policy.getClusterPreferences();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index f1799c4..1adfdbf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -96,7 +96,7 @@ public class Row implements MapWriter {
return null;
}
- Object getVal(String name, Object def) {
+ public Object getVal(String name, Object def) {
for (Cell cell : cells)
if (cell.name.equals(name)) {
return cell.val == null ? def : cell.val;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index f5e25cc..97b26fc 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -17,9 +17,12 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
@@ -29,15 +32,16 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.V2RequestSupport;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.ComputedType;
import org.apache.solr.client.solrj.cloud.autoscaling.Violation.ReplicaInfoAndErr;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
+import static java.util.Collections.emptySet;
import static java.util.Collections.unmodifiableSet;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -46,10 +50,41 @@ public class Suggestion {
public static final String coreidxsize = "INDEX.sizeInGB";
static final Map<String, ConditionType> validatetypes = new HashMap<>();
+ private static final String NULL = "";
+
+ @Target(ElementType.FIELD)
+ @Retention(RetentionPolicy.RUNTIME)
+ @interface Meta {
+ String name();
+
+ Class type();
+
+ String[] associatedPerNodeValue() default NULL;
+
+ String associatedPerReplicaValue() default NULL;
+
+ String[] enumVals() default NULL;
+
+ String[] wildCards() default NULL;
+
+ boolean isNodeSpecificVal() default false;
+
+ boolean isHidden() default false;
+
+ boolean isAdditive() default true;
+
+ double min() default -1d;
+
+ double max() default -1d;
+
+ String metricsKey() default NULL;
+
+ ComputedType[] computedValues() default ComputedType.NULL;
+ }
public static ConditionType getTagType(String name) {
ConditionType info = validatetypes.get(name);
- if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = ConditionType.LAZY;
+ if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = ConditionType.STRING;
if (info == null && name.startsWith(Clause.METRICS_PREFIX)) info = ConditionType.LAZY;
return info;
}
@@ -57,7 +92,7 @@ public class Suggestion {
private static Object getOperandAdjustedValue(Object val, Object original) {
if (original instanceof Clause.Condition) {
Clause.Condition condition = (Clause.Condition) original;
- if (condition.computationType == null && isIntegerEquivalent(val)) {
+ if (condition.computedType == null && isIntegerEquivalent(val)) {
if (condition.op == Operand.LESS_THAN) {
//replica : '<3'
val = val instanceof Long ?
@@ -119,26 +154,32 @@ public class Suggestion {
}
- public static final Map<String, String> tagVsPerReplicaVal = Stream.of(ConditionType.values())
- .filter(tag -> tag.perReplicaValue != null)
- .collect(Collectors.toMap(tag -> tag.tagName, tag -> tag.perReplicaValue));
-
/**
* Type details of each variable in policies
*/
public enum ConditionType {
- COLL("collection", String.class, null, null, null),
- SHARD("shard", String.class, null, null, null),
- REPLICA("replica", Double.class, null, 0L, null) {
+ @Meta(name = "collection",
+ type = String.class)
+ COLL(),
+ @Meta(
+ name = "shard",
+ type = String.class,
+ wildCards = {Policy.EACH, Policy.ANY})
+ SHARD(),
+
+ @Meta(name = "replica",
+ type = Double.class,
+ min = 0, max = -1,
+ computedValues = {ComputedType.EQUAL, ComputedType.PERCENT})
+ REPLICA() {
@Override
public Object validate(String name, Object val, boolean isRuleVal) {
return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
}
@Override
- public Operand getOperand(Operand expected, Object strVal, Clause.ComputationType computationType) {
-// if (computationType == Clause.ComputationType.EQUAL) return expected;
+ public Operand getOperand(Operand expected, Object strVal, ComputedType computedType) {
if (strVal instanceof String) {
String s = ((String) strVal).trim();
int hyphenIdx = s.indexOf('-');
@@ -152,24 +193,18 @@ public class Suggestion {
}
- if (expected == Operand.EQUAL && (computationType != null || !isIntegerEquivalent(strVal))) {
+ if (expected == Operand.EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) {
return Operand.RANGE_EQUAL;
}
- if (expected == Operand.NOT_EQUAL && (computationType != null || !isIntegerEquivalent(strVal)))
+ if (expected == Operand.NOT_EQUAL && (computedType != null || !isIntegerEquivalent(strVal)))
return Operand.RANGE_NOT_EQUAL;
return expected;
}
@Override
- public boolean supportComputed(Clause.ComputationType computedType, Clause clause) {
- if (computedType == Clause.ComputationType.PERCENT || computedType == Clause.ComputationType.EQUAL) return true;
- return false;
- }
-
- @Override
public String postValidate(Clause.Condition condition) {
- if (condition.computationType == Clause.ComputationType.EQUAL) {
+ if (condition.computedType == ComputedType.EQUAL) {
if (condition.getClause().tag != null &&
condition.getClause().tag.varType == NODE &&
condition.getClause().tag.op == Operand.WILDCARD) {
@@ -182,27 +217,53 @@ public class Suggestion {
}
@Override
- public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard) {
- if (cv.computationType == Clause.ComputationType.EQUAL) {
+ public Object computeValue(Policy.Session session, Clause.Condition cv, String collection, String shard, String node) {
+ if (cv.computedType == ComputedType.EQUAL) {
int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard);
if (relevantReplicasCount == 0) return 0;
return (double) session.matrix.size() / (double) relevantReplicasCount;
- } else if (cv.computationType == Clause.ComputationType.PERCENT) {
- int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard);
- if (relevantReplicasCount == 0) return 0;
- return (double) relevantReplicasCount * Clause.parseDouble(cv.name, cv.val).doubleValue() / 100;
+ } else if (cv.computedType == ComputedType.PERCENT) {
+ return ComputedType.PERCENT.compute(getRelevantReplicasCount(session, cv, collection, shard), cv);
} else {
- throw new IllegalArgumentException("Unsupported type " + cv.computationType);
+ throw new IllegalArgumentException("Unsupported type " + cv.computedType);
}
}
},
- PORT(ImplicitSnitch.PORT, Long.class, null, 1L, 65535L),
- IP_1("ip_1", Long.class, null, 0L, 255L),
- IP_2("ip_2", Long.class, null, 0L, 255L),
- IP_3("ip_3", Long.class, null, 0L, 255L),
- IP_4("ip_4", Long.class, null, 0L, 255L),
- FREEDISK(ImplicitSnitch.DISK, Double.class, null, 0d, Double.MAX_VALUE, coreidxsize, Boolean.TRUE,null) {
+ @Meta(name = ImplicitSnitch.PORT,
+ type = Long.class,
+ min = 1,
+ max = 65535)
+ PORT(),
+ @Meta(name = "ip_1",
+ type = Long.class,
+ min = 0,
+ max = 255)
+ IP_1(),
+ @Meta(name = "ip_2",
+ type = Long.class,
+ min = 0,
+ max = 255)
+ IP_2(),
+ @Meta(name = "ip_3",
+ type = Long.class,
+ min = 0,
+ max = 255)
+ IP_3(),
+ @Meta(name = "ip_4",
+ type = Long.class,
+ min = 0,
+ max = 255)
+ IP_4(),
+
+ @Meta(name = ImplicitSnitch.DISK,
+ type = Double.class,
+ min = 0,
+ isNodeSpecificVal = true,
+ associatedPerReplicaValue = coreidxsize,
+ associatedPerNodeValue = "totaldisk",
+ computedValues = ComputedType.PERCENT)
+ FREEDISK() {
@Override
public Object convertVal(Object val) {
Number value = (Number) super.validate(ImplicitSnitch.DISK, val, false);
@@ -213,6 +274,18 @@ public class Suggestion {
}
@Override
+ public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
+ if (condition.computedType == ComputedType.PERCENT) {
+ Row r = session.getNode(node);
+ if (r == null) return 0d;
+ return ComputedType.PERCENT.compute(r.getVal(TOTALDISK.tagName), condition);
+ }
+ throw new IllegalArgumentException("Unsupported type " + condition.computedType);
+ }
+
+
+
+ @Override
public int compareViolation(Violation v1, Violation v2) {
//TODO use tolerance compare
return Double.compare(
@@ -288,14 +361,38 @@ public class Suggestion {
cell.val = currFreeDisk + idxSize;
}
},
- CORE_IDX(coreidxsize, Double.class, null, 0d, Double.MAX_VALUE,null, false,"INDEX.sizeInBytes" ) {
+
+ @Meta(name = "totaldisk",
+ type = Double.class,
+ isHidden = true)
+ TOTALDISK() {
+ @Override
+ public Object convertVal(Object val) {
+ return FREEDISK.convertVal(val);
+ }
+ },
+
+ @Meta(name = coreidxsize,
+ type = Double.class,
+ isNodeSpecificVal = true,
+ isHidden = true,
+ min = 0,
+ metricsKey = "INDEX.sizeInBytes")
+ CORE_IDX() {
@Override
public Object convertVal(Object val) {
return FREEDISK.convertVal(val);
}
},
- NODE_ROLE(ImplicitSnitch.NODEROLE, String.class, Collections.singleton("overseer"), null, null),
- CORES(ImplicitSnitch.CORES, Long.class, null, 0L, Long.MAX_VALUE) {
+ @Meta(name = ImplicitSnitch.NODEROLE,
+ type = String.class,
+ enumVals = "overseer")
+ NODE_ROLE(),
+
+ @Meta(name = ImplicitSnitch.CORES,
+ type = Long.class,
+ min = 0)
+ CORES() {
@Override
public Object validate(String name, Object val, boolean isRuleVal) {
return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val);
@@ -335,12 +432,33 @@ public class Suggestion {
cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() - 1;
}
},
- SYSLOADAVG(ImplicitSnitch.SYSLOADAVG, Double.class, null, 0d, 100d),
- HEAPUSAGE(ImplicitSnitch.HEAPUSAGE, Double.class, null, 0d, null),
- NUMBER("NUMBER", Long.class, null, 0L, Long.MAX_VALUE),
- STRING("STRING", String.class, null, null, null),
- NODE("node", String.class, null, null, null) {
+ @Meta(name = ImplicitSnitch.SYSLOADAVG,
+ type = Double.class,
+ min = 0,
+ max = 100,
+ isNodeSpecificVal = true)
+ SYSLOADAVG(),
+
+ @Meta(name = ImplicitSnitch.HEAPUSAGE,
+ type = Double.class,
+ min = 0,
+ isNodeSpecificVal = true)
+ HEAPUSAGE(),
+ @Meta(name = "NUMBER",
+ type = Long.class,
+ min = 0)
+ NUMBER(),
+ @Meta(name = "STRING",
+ type = String.class,
+ wildCards = Policy.EACH)
+ STRING(),
+
+ @Meta(name = "node",
+ type = String.class,
+ isNodeSpecificVal = true,
+ wildCards = {Policy.ANY, Policy.EACH})
+ NODE() {
@Override
public void getSuggestions(SuggestionCtx ctx) {
if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return;
@@ -356,14 +474,17 @@ public class Suggestion {
}
- @Override
+ /*@Override
public void addViolatingReplicas(ViolationCtx ctx) {
for (Row r : ctx.allRows) {
if(r.node.equals(ctx.tagKey)) collectViolatingReplicas(ctx,r);
}
- }
+ }*/
},
- LAZY("LAZY", null, null, null, null) {
+
+ @Meta(name = "LAZY",
+ type = void.class)
+ LAZY() {
@Override
public Object validate(String name, Object val, boolean isRuleVal) {
return Clause.parseString(val);
@@ -374,38 +495,73 @@ public class Suggestion {
perNodeSuggestions(ctx);
}
},
- DISKTYPE(ImplicitSnitch.DISKTYPE, String.class,
- unmodifiableSet(new HashSet(Arrays.asList("ssd", "rotational"))), null, null) {
+
+ @Meta(name = ImplicitSnitch.DISKTYPE,
+ type = String.class,
+ enumVals = {"ssd", "rotational"})
+ DISKTYPE() {
@Override
public void getSuggestions(SuggestionCtx ctx) {
perNodeSuggestions(ctx);
}
};
- final Class type;
- final Set<String> vals;
- final Number min;
- final Number max;
- final Boolean additive;
public final String tagName;
+ public final Class type;
+ public Meta meta;
+
+ public final Set<String> vals;
+ public final Number min;
+ public final Number max;
+ public final Boolean additive;
+ public final boolean isHidden;
+ public final Set<String> wildCards;
public final String perReplicaValue;
+ public final Set<String> associatedPerNodeValues;
public final String metricsAttribute;
+ public final boolean isPerNodeValue;
+ public final Set<ComputedType> supportedComputedTypes;
+
+
+ ConditionType() {
+ try {
+ meta = ConditionType.class.getField(name()).getAnnotation(Meta.class);
+ if (meta == null) {
+ throw new RuntimeException("Invalid type, should have a @Meta annotation " + name());
+ }
+ } catch (NoSuchFieldException e) {
+ //cannot happen
+ }
+ this.tagName = meta.name();
+ this.type = meta.type();
+
+ this.vals = readSet(meta.enumVals());
+ this.max = readNum(meta.max());
+ this.min = readNum(meta.min());
+ this.perReplicaValue = readStr(meta.associatedPerReplicaValue());
+ this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue());
+ this.additive = meta.isAdditive();
+ this.metricsAttribute = readStr(meta.metricsKey());
+ this.isPerNodeValue = meta.isNodeSpecificVal();
+ this.supportedComputedTypes = meta.computedValues()[0] == ComputedType.NULL ?
+ emptySet() :
+ unmodifiableSet(new HashSet(Arrays.asList(meta.computedValues())));
+ this.isHidden = meta.isHidden();
+ this.wildCards = readSet(meta.wildCards());
+ }
- ConditionType(String tagName, Class type, Set<String> vals, Number min, Number max) {
- this(tagName, type, vals, min, max, null, Boolean.TRUE, null);
+ private String readStr(String s) {
+ return NULL.equals(s) ? null : s;
+ }
+ private Number readNum(double v) {
+ return v == -1 ? null :
+ (Number) validate(null, v, true);
}
- ConditionType(String tagName, Class type, Set<String> vals, Number min, Number max, String perReplicaValue,
- Boolean additive, String metricsAttribute) {
- this.tagName = tagName;
- this.type = type;
- this.vals = vals;
- this.min = min;
- this.max = max;
- this.perReplicaValue = perReplicaValue;
- this.additive = additive;
- this.metricsAttribute = metricsAttribute;
+ Set<String> readSet(String[] vals) {
+ if (NULL.equals(vals[0])) return emptySet();
+ return unmodifiableSet(new HashSet<>(Arrays.asList(vals)));
}
public void getSuggestions(SuggestionCtx ctx) {
@@ -414,11 +570,12 @@ public class Suggestion {
public void addViolatingReplicas(ViolationCtx ctx) {
for (Row row : ctx.allRows) {
+ if (ctx.clause.tag.varType.isPerNodeValue && !row.node.equals(ctx.tagKey)) continue;
collectViolatingReplicas(ctx, row);
}
}
- public Operand getOperand(Operand expected, Object val, Clause.ComputationType computationType) {
+ public Operand getOperand(Operand expected, Object val, ComputedType computedType) {
return expected;
}
@@ -461,7 +618,7 @@ public class Suggestion {
}
return num;
} else if (type == String.class) {
- if (isRuleVal && vals != null && !vals.contains(val))
+ if (isRuleVal && !vals.isEmpty() && !vals.contains(val))
throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(vals, ','));
return val;
} else {
@@ -485,24 +642,32 @@ public class Suggestion {
return Math.abs(v1.replicaCountDelta) < Math.abs(v2.replicaCountDelta) ? -1 : 1;
}
- public boolean supportComputed(Clause.ComputationType computedType, Clause clause) {
- return false;
- }
-
- public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard) {
+ public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
return condition.val;
}
}
private static void collectViolatingReplicas(ViolationCtx ctx, Row row) {
- row.forEachReplica(replica -> {
- if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
- if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
- if (!ctx.currentViolation.matchShard(replica.getShard())) return;
- if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
- return;
- ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
- });
+ if (ctx.clause.tag.varType.isPerNodeValue) {
+ row.forEachReplica(replica -> {
+ if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard())) {
+ ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica)
+ .withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
+ }
+ });
+ } else {
+ row.forEachReplica(replica -> {
+ if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
+ if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
+ if (!ctx.currentViolation.matchShard(replica.getShard())) return;
+ if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
+ return;
+ ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
+ });
+
+ }
+
+
}
private static int getRelevantReplicasCount(Policy.Session session, Clause.Condition cv, String collection, String shard) {
@@ -557,7 +722,12 @@ public class Suggestion {
}
}
+ /*public static final Map<String, String> tagVsPerReplicaVal = Stream.of(ConditionType.values())
+ .filter(tag -> tag.perReplicaValue != null)
+ .collect(Collectors.toMap(tag -> tag.tagName, tag -> tag.perReplicaValue));*/
static {
for (Suggestion.ConditionType t : Suggestion.ConditionType.values()) Suggestion.validatetypes.put(t.tagName, t);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 87b0abe..d4ab0e6 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -30,7 +30,7 @@ import org.apache.solr.common.util.Utils;
public class Violation implements MapWriter {
final String shard, coll, node;
final Object actualVal;
- final Double replicaCountDelta;//how far is the actual value from the expected value
+ final Double replicaCountDelta;//how many extra replicas
final Object tagKey;
private final int hash;
private final Clause clause;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 903a75c..92183b8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -58,6 +58,8 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyMap;
import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.TOTALDISK;
/**
*
@@ -215,18 +217,15 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
}
}
if (requestedTags.contains(ImplicitSnitch.DISKTYPE)) {
- metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", new Function<Object, Pair<String,Object>>() {
- @Override
- public Pair<String, Object> apply(Object o) {
- if("true".equals(String.valueOf(o))){
- return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational");
- }
- if("false".equals(String.valueOf(o))){
- return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd");
- }
- return new Pair<>(ImplicitSnitch.DISKTYPE,null);
-
+ metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", (Function<Object, Pair<String, Object>>) o -> {
+ if("true".equals(String.valueOf(o))){
+ return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational");
+ }
+ if("false".equals(String.valueOf(o))){
+ return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd");
}
+ return new Pair<>(ImplicitSnitch.DISKTYPE,null);
+
});
}
if (!metricsKeyVsTag.isEmpty()) {
@@ -239,6 +238,10 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
groups.add("solr.node");
prefixes.add("CONTAINER.fs.usableSpace");
}
+ if (requestedTags.contains(TOTALDISK.tagName)) {
+ groups.add("solr.node");
+ prefixes.add("CONTAINER.fs.totalSpace");
+ }
if (requestedTags.contains(CORES)) {
groups.add("solr.core");
prefixes.add("CORE.coreName");
@@ -260,9 +263,13 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
try {
SimpleSolrResponse rsp = snitchContext.invoke(solrNode, CommonParams.METRICS_PATH, params);
Map m = rsp.nl.asMap(4);
- if (requestedTags.contains(DISK)) {
+ if (requestedTags.contains(FREEDISK.tagName)) {
Object n = Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.usableSpace");
- if (n != null) ctx.getTags().put(DISK, Suggestion.getTagType(DISK).convertVal(n));
+ if (n != null) ctx.getTags().put(FREEDISK.tagName, FREEDISK.convertVal(n));
+ }
+ if (requestedTags.contains(TOTALDISK.tagName)) {
+ Object n = Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.totalSpace");
+ if (n != null) ctx.getTags().put(TOTALDISK.tagName, TOTALDISK.convertVal(n));
}
if (requestedTags.contains(CORES)) {
int count = 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index 1fba397..fc57e3d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -107,6 +107,11 @@ public interface MapWriter extends MapSerializable {
return this;
}
+ default EntryWriter putStringIfNotNull(String k, Object v) throws IOException {
+ if(v != null) put(k,String.valueOf(v));
+ return this;
+ }
+
default EntryWriter put(String k, int v) throws IOException {
put(k, (Integer) v);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/738dadca/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 2439d56..5147049 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -212,30 +212,30 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(Operand.RANGE_EQUAL, REPLICA.getOperand(Operand.EQUAL, "2.01", null));
assertEquals(Operand.RANGE_NOT_EQUAL, REPLICA.getOperand(Operand.NOT_EQUAL, "2.01", null));
- Clause clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '1.23', node:'#ANY'}"));
+ Clause clause = Clause.create("{replica: '1.23', node:'#ANY'}");
assertTrue(clause.getReplica().isPass(2));
assertTrue(clause.getReplica().isPass(1));
assertFalse(clause.getReplica().isPass(0));
assertFalse(clause.getReplica().isPass(3));
- clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '<1.23', node:'#ANY'}"));
+ clause = Clause.create("{replica: '<1.23', node:'#ANY'}");
assertTrue(clause.getReplica().isPass(1));
assertTrue(clause.getReplica().isPass(0));
assertFalse(clause.getReplica().isPass(2));
- clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '!1.23', node:'#ANY'}"));
+ clause = Clause.create("{replica: '!1.23', node:'#ANY'}");
assertFalse(clause.getReplica().isPass(2));
assertFalse(clause.getReplica().isPass(1));
assertTrue(clause.getReplica().isPass(0));
assertTrue(clause.getReplica().isPass(3));
- clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: 1.23, node:'#ANY'}"));
+ clause = Clause.create("{replica: 1.23, node:'#ANY'}");
assertTrue(clause.getReplica().isPass(2));
assertTrue(clause.getReplica().isPass(1));
assertFalse(clause.getReplica().isPass(0));
assertFalse(clause.getReplica().isPass(3));
- clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '33%', node:'#ANY'}"));
+ clause = Clause.create("{replica: '33%', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.getReplica().op);
clause = clause.getSealedClause(condition -> {
if (condition.name.equals("replica")) {
@@ -245,7 +245,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
});
assertTrue( clause.getReplica().isPass(2));
- clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '3 - 5', node:'#ANY'}"));
+ clause = Clause.create("{replica: '3 - 5', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.getReplica().getOperand());
RangeVal range = (RangeVal) clause.getReplica().getValue();
assertEquals(3.0 , range.min);
@@ -261,18 +261,27 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(Double.valueOf(0.0), clause.replica.delta(4));
expectThrows(IllegalArgumentException.class,
- () -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '-33%', node:'#ANY'}")));
+ () -> Clause.create("{replica: '-33%', node:'#ANY'}"));
expectThrows(IllegalArgumentException.class,
- () -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: 'x%', node:'#ANY'}")));
+ () -> Clause.create("{replica: 'x%', node:'#ANY'}"));
expectThrows(IllegalArgumentException.class,
- () -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '20%-33%', node:'#ANY'}")));
+ () -> Clause.create("{replica: '20%-33%', node:'#ANY'}"));
- clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '#EQUAL', shard:'#EACH', node:'#ANY'}"));
+ clause = Clause.create("{replica: '#EQUAL', shard:'#EACH', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.replica.op);
- clause = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '#EQUAL', node:'#ANY'}"));
+ clause = Clause.create("{replica: '#EQUAL', node:'#ANY'}");
assertEquals(Operand.RANGE_EQUAL, clause.replica.op);
expectThrows(IllegalArgumentException.class,
- () -> Clause.create((Map<String, Object>) Utils.fromJSONString("{replica: '#EQUAL', node:'node_1'}")));
+ () -> Clause.create("{replica: '#EQUAL', node:'node_1'}"));
+ clause = Clause.create("{replica : 0, freedisk:'<20%'}");
+ assertEquals(clause.tag.computedType, Clause.ComputedType.PERCENT);
+ assertEquals(clause.tag.op, Operand.LESS_THAN);
+ expectThrows(IllegalArgumentException.class,
+ () -> Clause.create("{replica : 0, INDEX.sizeInGB:'>300'}"));
+ expectThrows(IllegalArgumentException.class,
+ () -> Clause.create("{replica:'<3', shard: '#ANV', node:'#ANY'}"));
+ expectThrows(IllegalArgumentException.class,
+ () -> Clause.create("{replica:'<3', shard: '#EACH', node:'#E4CH'}"));
}
@@ -428,36 +437,36 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
public void testOperands() {
- Clause c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:'<2', node:'#ANY'}"));
+ Clause c = Clause.create("{replica:'<2', node:'#ANY'}");
assertFalse(c.replica.isPass(3));
assertFalse(c.replica.isPass(2));
assertTrue(c.replica.isPass(1));
assertEquals("{\"replica\":\"<2.0\"}", c.replica.toString());
- c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:'>2', node:'#ANY'}"));
+ c = Clause.create("{replica:'>2', node:'#ANY'}");
assertTrue(c.replica.isPass(3));
assertFalse(c.replica.isPass(2));
assertFalse(c.replica.isPass(1));
assertEquals("{\"replica\":\">2.0\"}", c.replica.toString());
- c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}"));
+ c = Clause.create("{replica:0, nodeRole:'!overseer'}");
assertTrue(c.tag.isPass("OVERSEER"));
assertFalse(c.tag.isPass("overseer"));
- c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'<12.7'}"));
+ c = Clause.create("{replica:0, sysLoadAvg:'<12.7'}");
assertTrue(c.tag.isPass("12.6"));
assertTrue(c.tag.isPass(12.6d));
assertFalse(c.tag.isPass("12.9"));
assertFalse(c.tag.isPass(12.9d));
- c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'>12.7'}"));
+ c = Clause.create("{replica:0, sysLoadAvg:'>12.7'}");
assertTrue(c.tag.isPass("12.8"));
assertTrue(c.tag.isPass(12.8d));
assertFalse(c.tag.isPass("12.6"));
assertFalse(c.tag.isPass(12.6d));
- c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, 'metrics:x:y:z':'>12.7'}"));
+ c = Clause.create("{replica:0, 'metrics:x:y:z':'>12.7'}");
assertTrue(c.tag.val instanceof String);
assertTrue(c.tag.isPass("12.8"));
assertTrue(c.tag.isPass(12.8d));
@@ -465,7 +474,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertFalse(c.tag.isPass(12.6d));
try {
- c = Clause.create((Map<String, Object>) Utils.fromJSONString("{replica:0, 'ip_1':'<30%'}"));
+ c = Clause.create("{replica:0, 'ip_1':'<30%'}");
fail("Expected exception");
} catch (Exception e) {
assertTrue(e.getMessage().contains("'%' is not allowed for variable : 'ip_1'"));
@@ -1099,7 +1108,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
Policy.Session session;
session = policy.createSession(getSolrCloudManager(nodeValues, clusterState));
- List<Row> l = session.getSorted();
+ List<Row> l = session.getSortedNodes();
assertEquals("node1", l.get(0).node);
assertEquals("node3", l.get(1).node);
assertEquals("node4", l.get(2).node);
@@ -1387,7 +1396,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"{replica:0, 'nodeRole':'overseer','strict':false}," +
"{'replica':'<1','node':'node3'}," +
"{'replica':'<2','node':'#ANY','shard':'#EACH'}," +
- "{'replica':'<3','shard':'#EACH','sysprop.rack':'#ANY'}" +
+ "{'replica':'<3','shard':'#EACH','sysprop.rack':'#EACH'}" +
"]" +
"}" +
"}";
@@ -1871,6 +1880,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
assertEquals(1, violations.size());
assertEquals(4, violations.get(0).getViolatingReplicas().size());
+ for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
+ assertEquals(500d, r.delta, 0.1);
+
+ }
List<Suggester.SuggestionInfo> l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(3, l.size());
Map m = l.get(0).toMap(new LinkedHashMap<>());
@@ -2873,4 +2886,66 @@ public void testUtilizeNodeFailure2() throws Exception {
}
+ public void testFreediskPercentage(){
+ String dataproviderdata = "{" +
+ " 'liveNodes': [" +
+ " 'node1:8983'," +
+ " 'node2:8984'," +
+ " 'node3:8985'" +
+ " ]," +
+ " 'replicaInfo': {" +
+ " 'node1:8983': {" +
+ " 'c1': {" +
+ " 's1': [" +
+ " {'r1': {'type': 'NRT'}}," +
+ " {'r2': {'type': 'NRT'}}" +
+ " ]," +
+ " 's2': [" +
+ " {'r1': {'type': 'NRT'}}," +
+ " {'r2': {'type': 'NRT'}}" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " 'nodeValues': {" +
+ " 'node1:8983': {" +
+ " 'cores': 4," +
+ " 'freedisk': 230," +
+ " 'totaldisk': 800," +
+ " 'port': 8983" +
+ " }," +
+ " 'node2:8984': {" +
+ " 'cores': 0," +
+ " 'freedisk': 1000," +
+ " 'totaldisk': 1200," +
+ " 'port': 8984" +
+ " }," +
+ " 'node3:8985': {" +
+ " 'cores': 0," +
+ " 'freedisk': 1500," +
+ " 'totaldisk': 1700," +
+ " 'port': 8985" +
+ " }" +
+ " }" +
+ "}";
+ String autoScalingjson = "{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 3}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, freedisk : '<30%'}" +
+ " ]" +
+ "}";
+ AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
+ List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) Utils.fromJSONString(dataproviderdata))).getViolations();
+ assertEquals(1, violations.size());
+ assertEquals(4, violations.get(0).getViolatingReplicas().size());
+ for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
+ assertEquals(10.0d, r.delta.doubleValue(), 0.1);
+ }
+
+ }
+
+
}