You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/08/06 04:15:35 UTC
[02/48] lucene-solr:jira/http2: SOLR-12601: Refactor the autoscaling
package to improve readability
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
new file mode 100644
index 0000000..8df74bf
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+
+import static java.util.Collections.emptySet;
+import static java.util.Collections.unmodifiableSet;
+
+/**
+ * A Variable Type used in Autoscaling policy rules. Each variable type may have unique implementation
+ * of functionalities
+ */
+public interface Variable {
+ String NULL = "";
+ String coreidxsize = "INDEX.sizeInGB";
+
+ default boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+ return op.match(val, validate(name, inputVal, false)) == Clause.TestStatus.PASS;
+ }
+ default Object convertVal(Object val) {
+ return val;
+ }
+
+ default void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode) {
+ }
+
+ default void addViolatingReplicas(Violation.Ctx ctx) {
+ for (Row row : ctx.allRows) {
+ if (ctx.clause.tag.varType.meta.isNodeSpecificVal() && !row.node.equals(ctx.tagKey)) continue;
+ Violation.collectViolatingReplicas(ctx, row);
+ }
+ }
+
+ void getSuggestions(Suggestion.Ctx ctx) ;
+
+ default Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
+ return condition.val;
+ }
+
+ int compareViolation(Violation v1, Violation v2);
+
+ default void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
+ }
+
+ default String postValidate(Clause.Condition condition) {
+ return null;
+ }
+
+ default Operand getOperand(Operand expected, Object strVal, Clause.ComputedType computedType) {
+ return expected;
+ }
+
+ Object validate(String name, Object val, boolean isRuleVal);
+
+ /**
+ * Type details of each variable in policies
+ */
+ public enum Type implements Variable {
+ @Meta(name = "withCollection", type = String.class, isNodeSpecificVal = true, implementation = WithCollectionVariable.class)
+ WITH_COLLECTION(),
+
+ @Meta(name = "collection",
+ type = String.class)
+ COLL(),
+ @Meta(
+ name = "shard",
+ type = String.class,
+ wildCards = {Policy.EACH, Policy.ANY})
+ SHARD(),
+
+ @Meta(name = "replica",
+ type = Double.class,
+ min = 0, max = -1,
+ implementation = ReplicaVariable.class,
+ computedValues = {Clause.ComputedType.EQUAL, Clause.ComputedType.PERCENT, Clause.ComputedType.ALL})
+ REPLICA(),
+ @Meta(name = ImplicitSnitch.PORT,
+ type = Long.class,
+ min = 1,
+ max = 65535,
+ supportArrayVals = true,
+ wildCards = Policy.EACH
+ )
+ PORT(),
+ @Meta(name = "ip_1",
+ type = Long.class,
+ min = 0,
+ max = 255,
+ supportArrayVals = true,
+ wildCards = Policy.EACH)
+ IP_1(),
+ @Meta(name = "ip_2",
+ type = Long.class,
+ min = 0,
+ max = 255,
+ supportArrayVals = true,
+ wildCards = Policy.EACH)
+ IP_2(),
+ @Meta(name = "ip_3",
+ type = Long.class,
+ min = 0,
+ max = 255,
+ supportArrayVals = true,
+ wildCards = Policy.EACH)
+ IP_3(),
+ @Meta(name = "ip_4",
+ type = Long.class,
+ min = 0,
+ max = 255,
+ supportArrayVals = true,
+ wildCards = Policy.EACH)
+ IP_4(),
+ @Meta(name = ImplicitSnitch.DISK,
+ type = Double.class,
+ min = 0,
+ isNodeSpecificVal = true,
+ associatedPerReplicaValue = Variable.coreidxsize,
+ associatedPerNodeValue = "totaldisk",
+ implementation = FreeDiskVariable.class,
+ computedValues = Clause.ComputedType.PERCENT)
+ FREEDISK(),
+
+ @Meta(name = "totaldisk",
+ type = Double.class,
+ isHidden = true, implementation = VariableBase.TotalDiskVariable.class)
+ TOTALDISK(),
+
+ @Meta(name = Variable.coreidxsize,
+ type = Double.class,
+ isNodeSpecificVal = true,
+ isHidden = true,
+ min = 0,
+ implementation = VariableBase.CoreIndexSizeVariable.class,
+ metricsKey = "INDEX.sizeInBytes")
+ CORE_IDX(),
+ @Meta(name = ImplicitSnitch.NODEROLE,
+ type = String.class,
+ enumVals = "overseer")
+ NODE_ROLE(),
+
+ @Meta(name = ImplicitSnitch.CORES,
+ type = Long.class,
+ min = 0,
+ implementation = CoresVariable.class)
+ CORES(),
+
+ @Meta(name = ImplicitSnitch.SYSLOADAVG,
+ type = Double.class,
+ min = 0,
+ max = 100,
+ isNodeSpecificVal = true)
+ SYSLOADAVG(),
+
+ @Meta(name = ImplicitSnitch.HEAPUSAGE,
+ type = Double.class,
+ min = 0,
+ isNodeSpecificVal = true)
+ HEAPUSAGE(),
+ @Meta(name = "NUMBER",
+ type = Long.class,
+ min = 0)
+ NUMBER(),
+
+ @Meta(name = "STRING",
+ type = String.class,
+ wildCards = Policy.EACH,
+ supportArrayVals = true)
+ STRING(),
+
+ @Meta(name = "node",
+ type = String.class,
+ isNodeSpecificVal = true,
+ wildCards = {Policy.ANY, Policy.EACH},
+ implementation = NodeVariable.class,
+ supportArrayVals = true)
+ NODE(),
+
+ @Meta(name = "LAZY",
+ type = void.class,
+ implementation = VariableBase.LazyVariable.class)
+ LAZY(),
+
+ @Meta(name = ImplicitSnitch.DISKTYPE,
+ type = String.class,
+ enumVals = {"ssd", "rotational"},
+ implementation = VariableBase.DiskTypeVariable.class,
+ supportArrayVals = true)
+ DISKTYPE();
+
+ public final String tagName;
+ public final Class type;
+ public Meta meta;
+
+ public final Set<String> vals;
+ public final Number min;
+ public final Number max;
+ public final Boolean additive;
+ public final Set<String> wildCards;
+ public final String perReplicaValue;
+ public final Set<String> associatedPerNodeValues;
+ public final String metricsAttribute;
+ public final Set<Clause.ComputedType> supportedComputedTypes;
+ final Variable impl;
+
+
+ Type() {
+ try {
+ meta = Type.class.getField(name()).getAnnotation(Meta.class);
+ if (meta == null) {
+ throw new RuntimeException("Invalid type, should have a @Meta annotation " + name());
+ }
+ } catch (NoSuchFieldException e) {
+ //cannot happen
+ }
+ impl= VariableBase.loadImpl(meta, this);
+
+ this.tagName = meta.name();
+ this.type = meta.type();
+
+ this.vals = readSet(meta.enumVals());
+ this.max = readNum(meta.max());
+ this.min = readNum(meta.min());
+ this.perReplicaValue = readStr(meta.associatedPerReplicaValue());
+ this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue());
+ this.additive = meta.isAdditive();
+ this.metricsAttribute = readStr(meta.metricsKey());
+ this.supportedComputedTypes = meta.computedValues()[0] == Clause.ComputedType.NULL ?
+ emptySet() :
+ unmodifiableSet(new HashSet(Arrays.asList(meta.computedValues())));
+ this.wildCards = readSet(meta.wildCards());
+
+ }
+
+ public String getTagName() {
+ return meta.name();
+ }
+
+ private String readStr(String s) {
+ return NULL.equals(s) ? null : s;
+ }
+
+ private Number readNum(double v) {
+ return v == -1 ? null :
+ (Number) validate(null, v, true);
+ }
+
+ Set<String> readSet(String[] vals) {
+ if (NULL.equals(vals[0])) return emptySet();
+ return unmodifiableSet(new HashSet<>(Arrays.asList(vals)));
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ impl.getSuggestions(ctx);
+ }
+
+ @Override
+ public void addViolatingReplicas(Violation.Ctx ctx) {
+ impl.addViolatingReplicas(ctx);
+ }
+
+ public Operand getOperand(Operand expected, Object val, Clause.ComputedType computedType) {
+ return impl.getOperand(expected, val, computedType);
+ }
+
+
+ public Object convertVal(Object val) {
+ return impl.convertVal(val);
+ }
+
+ public String postValidate(Clause.Condition condition) {
+ return impl.postValidate(condition);
+ }
+
+ public Object validate(String name, Object val, boolean isRuleVal) {
+ return impl.validate(name, val, isRuleVal);
+ }
+
+ /**
+ * Simulate a replica addition to a node in the cluster
+ */
+ public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode) {
+ impl.projectAddReplica(cell, ri, opCollector, strictMode);
+ }
+
+ public void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector) {
+ impl.projectRemoveReplica(cell, ri, opCollector);
+ }
+
+ @Override
+ public int compareViolation(Violation v1, Violation v2) {
+ return impl.compareViolation(v1, v2);
+ }
+
+ @Override
+ public Object computeValue(Policy.Session session, Clause.Condition condition, String collection, String shard, String node) {
+ return impl.computeValue(session, condition, collection, shard, node);
+ }
+
+ @Override
+ public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+ return impl.match(inputVal, op, val, name, row);
+ }
+ }
+
+ @Target(ElementType.FIELD)
+ @Retention(RetentionPolicy.RUNTIME)
+ @interface Meta {
+ String name();
+
+ Class type();
+
+ String[] associatedPerNodeValue() default NULL;
+
+ String associatedPerReplicaValue() default NULL;
+
+ String[] enumVals() default NULL;
+
+ String[] wildCards() default NULL;
+
+ boolean isNodeSpecificVal() default false;
+
+ boolean isHidden() default false;
+
+ boolean isAdditive() default true;
+
+ double min() default -1d;
+
+ double max() default -1d;
+
+ boolean supportArrayVals() default false;
+
+ String metricsKey() default NULL;
+
+ Class implementation() default void.class;
+
+ Clause.ComputedType[] computedValues() default Clause.ComputedType.NULL;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
new file mode 100644
index 0000000..ad2b43b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.util.StrUtils;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.parseString;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.perNodeSuggestions;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
+
+public class VariableBase implements Variable {
+ final Type varType;
+
+ public VariableBase(Type type) {
+ this.varType = type;
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ perNodeSuggestions(ctx);
+ }
+
+ static Object getOperandAdjustedValue(Object val, Object original) {
+ if (original instanceof Clause.Condition) {
+ Clause.Condition condition = (Clause.Condition) original;
+ if (condition.computedType == null && isIntegerEquivalent(val)) {
+ if (condition.op == Operand.LESS_THAN) {
+ //replica : '<3'
+ val = val instanceof Long ?
+ (Long) val - 1 :
+ (Double) val - 1;
+ } else if (condition.op == Operand.GREATER_THAN) {
+ //replica : '>4'
+ val = val instanceof Long ?
+ (Long) val + 1 :
+ (Double) val + 1;
+ }
+ }
+ }
+ return val;
+ }
+
+ static boolean isIntegerEquivalent(Object val) {
+ if (val instanceof Number) {
+ Number number = (Number) val;
+ return Math.ceil(number.doubleValue()) == Math.floor(number.doubleValue());
+ } else if (val instanceof String) {
+ try {
+ double dval = Double.parseDouble((String) val);
+ return Math.ceil(dval) == Math.floor(dval);
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+
+ }
+
+ public static Type getTagType(String name) {
+ Type info = validatetypes.get(name);
+ if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = Type.STRING;
+ if (info == null && name.startsWith(Clause.METRICS_PREFIX)) info = Type.LAZY;
+ return info;
+ }
+
+ static Variable loadImpl(Meta meta, Type t) {
+ Class implementation = meta.implementation();
+ if (implementation == void.class) implementation = VariableBase.class;
+ try {
+ return (Variable) implementation.getConstructor(Type.class).newInstance(t);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate: " + implementation.getName(), e);
+ }
+ }
+
+ @Override
+ public int compareViolation(Violation v1, Violation v2) {
+ if (v2.replicaCountDelta == null || v1.replicaCountDelta == null) return 0;
+ if (Math.abs(v1.replicaCountDelta) == Math.abs(v2.replicaCountDelta)) return 0;
+ return Math.abs(v1.replicaCountDelta) < Math.abs(v2.replicaCountDelta) ? -1 : 1;
+ }
+
+ @Override
+ public Object validate(String name, Object val, boolean isRuleVal) {
+ if (val instanceof Clause.Condition) {
+ Clause.Condition condition = (Clause.Condition) val;
+ val = condition.op.readRuleValue(condition);
+ if (val != condition.val) return val;
+ }
+ if (name == null) name = this.varType.tagName;
+ if (varType.type == Double.class) {
+ Double num = Clause.parseDouble(name, val);
+ if (isRuleVal) {
+ if (varType.min != null)
+ if (Double.compare(num, varType.min.doubleValue()) == -1)
+ throw new RuntimeException(name + ": " + val + " must be greater than " + varType.min);
+ if (varType.max != null)
+ if (Double.compare(num, varType.max.doubleValue()) == 1)
+ throw new RuntimeException(name + ": " + val + " must be less than " + varType.max);
+ }
+ return num;
+ } else if (varType.type == Long.class) {
+ Long num = Clause.parseLong(name, val);
+ if (isRuleVal) {
+ if (varType.min != null)
+ if (num < varType.min.longValue())
+ throw new RuntimeException(name + ": " + val + " must be greater than " + varType.min);
+ if (varType.max != null)
+ if (num > varType.max.longValue())
+ throw new RuntimeException(name + ": " + val + " must be less than " + varType.max);
+ }
+ return num;
+ } else if (varType.type == String.class) {
+ if (isRuleVal && !varType.vals.isEmpty() && !varType.vals.contains(val))
+ throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(varType.vals, ','));
+ return val;
+ } else {
+ throw new RuntimeException("Invalid type ");
+ }
+ }
+
+ public static class TotalDiskVariable extends VariableBase {
+ public TotalDiskVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public Object convertVal(Object val) {
+ return FREEDISK.convertVal(val);
+ }
+ }
+
+ public static class CoreIndexSizeVariable extends VariableBase {
+ public CoreIndexSizeVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public Object convertVal(Object val) {
+ return FREEDISK.convertVal(val);
+ }
+ }
+
+ public static class LazyVariable extends VariableBase {
+ public LazyVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public Object validate(String name, Object val, boolean isRuleVal) {
+ return parseString(val);
+ }
+
+ @Override
+ public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+ return op.match(parseString(val), parseString(inputVal)) == Clause.TestStatus.PASS;
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ perNodeSuggestions(ctx);
+ }
+ }
+
+ public static class DiskTypeVariable extends VariableBase {
+ public DiskTypeVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ perNodeSuggestions(ctx);
+ }
+
+
+ }
+
+ private static Map<String, Type> validatetypes;
+
+ static {
+ validatetypes = new HashMap<>();
+ for (Type t : Type.values())
+ validatetypes.put(t.tagName, t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 7b0f0f3..2f81291 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
+import java.util.function.Function;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.Utils;
@@ -47,6 +48,29 @@ public class Violation implements MapWriter {
hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
}
+ static void collectViolatingReplicas(Ctx ctx, Row row) {
+ if (ctx.clause.tag.varType.meta.isNodeSpecificVal()) {
+ row.forEachReplica(replica -> {
+ if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard())) {
+ ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica)
+ .withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
+ }
+ });
+ } else {
+ row.forEachReplica(replica -> {
+ if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
+ if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
+ if(!ctx.currentViolation.getClause().matchShard(replica.getShard(), ctx.currentViolation.shard)) return;
+ if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
+ return;
+ ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
+ });
+
+ }
+
+
+ }
+
public Violation addReplica(ReplicaInfoAndErr r) {
replicaInfoAndErrs.add(r);
return this;
@@ -144,4 +168,29 @@ public class Violation implements MapWriter {
});
ew.put("clause", getClause());
}
+
+ static class Ctx {
+ final Function<Clause.Condition, Object> evaluator;
+ String tagKey;
+ Clause clause;
+ ReplicaCount count;
+ Violation currentViolation;
+ List<Row> allRows;
+ List<Violation> allViolations = new ArrayList<>();
+
+ public Ctx(Clause clause, List<Row> allRows, Function<Clause.Condition, Object> evaluator) {
+ this.allRows = allRows;
+ this.clause = clause;
+ this.evaluator = evaluator;
+ }
+
+ public Ctx reset(String tagKey, ReplicaCount count, Violation currentViolation) {
+ this.tagKey = tagKey;
+ this.count = count;
+ this.currentViolation = currentViolation;
+ allViolations.add(currentViolation);
+ this.clause = currentViolation.getClause();
+ return this;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java
deleted file mode 100644
index 989a087..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.cloud.autoscaling;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Consumer;
-
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.util.Pair;
-
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
-
-/**
- * Implements the 'withCollection' variable type
- */
-public class WithCollectionVarType implements VarType {
- @Override
- public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
- Map<String, String> withCollectionMap = (Map<String, String>) inputVal;
- if (withCollectionMap == null || withCollectionMap.isEmpty()) return true;
-
- Set<String> uniqueColls = new HashSet<>();
- row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
-
- for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
- if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue())) return false;
- }
-
- return true;
- }
-
- public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode) {
- if (strictMode) {
- // we do not want to add a replica of the 'withCollection' in strict mode
- return;
- }
-
- Map<String, String> withCollectionMap = (Map<String, String>) cell.val;
- if (withCollectionMap == null || withCollectionMap.isEmpty()) return;
-
- Set<String> uniqueColls = new HashSet<>();
- Row row = cell.row;
- row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
-
- for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
- if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue())) {
- String withCollection = e.getValue();
-
- opCollector.accept(new Row.OperationInfo(withCollection, "shard1", row.node, cell.name, true, Replica.Type.NRT));
- }
- }
- }
-
- @Override
- public int compareViolation(Violation v1, Violation v2) {
- return Integer.compare(v1.getViolatingReplicas().size(), v2.getViolatingReplicas().size());
- }
-
- public void addViolatingReplicas(Suggestion.ViolationCtx ctx) {
- String node = ctx.currentViolation.node;
- for (Row row : ctx.allRows) {
- if (node.equals(row.node)) {
- Map<String, String> withCollectionMap = (Map<String, String>) row.getVal("withCollection");
- if (withCollectionMap != null) {
- row.forEachReplica(r -> {
- String withCollection = withCollectionMap.get(r.getCollection());
- if (withCollection != null) {
- // test whether this row has at least 1 replica of withCollection, else there is a violation
- Set<String> uniqueCollections = new HashSet<>();
- row.forEachReplica(replicaInfo -> uniqueCollections.add(replicaInfo.getCollection()));
- if (!uniqueCollections.contains(withCollection)) {
- ctx.currentViolation.addReplica(new Violation.ReplicaInfoAndErr(r).withDelta(1.0d));
- }
- }
- });
- ctx.currentViolation.replicaCountDelta = (double) ctx.currentViolation.getViolatingReplicas().size();
- }
- }
- }
- }
-
- @Override
- public void getSuggestions(Suggestion.SuggestionCtx ctx) {
- if (ctx.violation.getViolatingReplicas().isEmpty()) return;
-
- Map<String, Object> nodeValues = ctx.session.nodeStateProvider.getNodeValues(ctx.violation.node, Collections.singleton("withCollection"));
- Map<String, String> withCollectionsMap = (Map<String, String>) nodeValues.get("withCollection");
- if (withCollectionsMap == null) return;
-
- Set<String> uniqueCollections = new HashSet<>();
- for (Violation.ReplicaInfoAndErr replicaInfoAndErr : ctx.violation.getViolatingReplicas()) {
- uniqueCollections.add(replicaInfoAndErr.replicaInfo.getCollection());
- }
-
- collectionLoop:
- for (String collection : uniqueCollections) {
- String withCollection = withCollectionsMap.get(collection);
- if (withCollection == null) continue;
-
- // can we find a node from which we can move a replica of the `withCollection`
- // without creating another violation?
- for (Row row : ctx.session.matrix) {
- if (ctx.violation.node.equals(row.node)) continue; // filter the violating node
-
- Set<String> hostedCollections = new HashSet<>();
- row.forEachReplica(replicaInfo -> hostedCollections.add(replicaInfo.getCollection()));
-
- if (hostedCollections.contains(withCollection) && !hostedCollections.contains(collection)) {
- // find the candidate replicas that we can move
- List<ReplicaInfo> movableReplicas = new ArrayList<>();
- row.forEachReplica(replicaInfo -> {
- if (replicaInfo.getCollection().equals(withCollection)) {
- movableReplicas.add(replicaInfo);
- }
- });
-
- for (ReplicaInfo toMove : movableReplicas) {
- // candidate source node for a move replica operation
- Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
- .forceOperation(true)
- .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
- .hint(Suggester.Hint.SRC_NODE, row.node)
- .hint(Suggester.Hint.REPLICA, toMove.getName())
- .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
- if (ctx.addSuggestion(suggester) != null)
- continue collectionLoop; // one suggestion is enough for this collection
- }
- }
- }
-
- // we could not find a valid move, so we suggest adding a replica
- Suggester suggester = ctx.session.getSuggester(ADDREPLICA)
- .forceOperation(true)
- .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
- .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
- ctx.addSuggestion(suggester);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
new file mode 100644
index 0000000..b295aee
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.util.Pair;
+
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+
+/**
+ * Implements the 'withCollection' variable type
+ */
+public class WithCollectionVariable extends VariableBase {
+
+ public WithCollectionVariable(Type type) {
+ super(type);
+ }
+
+ @Override
+ public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+ Map<String, String> withCollectionMap = (Map<String, String>) inputVal;
+ if (withCollectionMap == null || withCollectionMap.isEmpty()) return true;
+
+ Set<String> uniqueColls = new HashSet<>();
+ row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
+
+ for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
+ if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue())) return false;
+ }
+
+ return true;
+ }
+
+ public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode) {
+ if (strictMode) {
+ // we do not want to add a replica of the 'withCollection' in strict mode
+ return;
+ }
+
+ Map<String, String> withCollectionMap = (Map<String, String>) cell.val;
+ if (withCollectionMap == null || withCollectionMap.isEmpty()) return;
+
+ Set<String> uniqueColls = new HashSet<>();
+ Row row = cell.row;
+ row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
+
+ for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
+ if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue())) {
+ String withCollection = e.getValue();
+
+ opCollector.accept(new Row.OperationInfo(withCollection, "shard1", row.node, cell.name, true, Replica.Type.NRT));
+ }
+ }
+ }
+
+ @Override
+ public int compareViolation(Violation v1, Violation v2) {
+ return Integer.compare(v1.getViolatingReplicas().size(), v2.getViolatingReplicas().size());
+ }
+
+ public void addViolatingReplicas(Violation.Ctx ctx) {
+ String node = ctx.currentViolation.node;
+ for (Row row : ctx.allRows) {
+ if (node.equals(row.node)) {
+ Map<String, String> withCollectionMap = (Map<String, String>) row.getVal("withCollection");
+ if (withCollectionMap != null) {
+ row.forEachReplica(r -> {
+ String withCollection = withCollectionMap.get(r.getCollection());
+ if (withCollection != null) {
+ // test whether this row has at least 1 replica of withCollection, else there is a violation
+ Set<String> uniqueCollections = new HashSet<>();
+ row.forEachReplica(replicaInfo -> uniqueCollections.add(replicaInfo.getCollection()));
+ if (!uniqueCollections.contains(withCollection)) {
+ ctx.currentViolation.addReplica(new Violation.ReplicaInfoAndErr(r).withDelta(1.0d));
+ }
+ }
+ });
+ ctx.currentViolation.replicaCountDelta = (double) ctx.currentViolation.getViolatingReplicas().size();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void getSuggestions(Suggestion.Ctx ctx) {
+ if (ctx.violation.getViolatingReplicas().isEmpty()) return;
+
+ Map<String, Object> nodeValues = ctx.session.nodeStateProvider.getNodeValues(ctx.violation.node, Collections.singleton("withCollection"));
+ Map<String, String> withCollectionsMap = (Map<String, String>) nodeValues.get("withCollection");
+ if (withCollectionsMap == null) return;
+
+ Set<String> uniqueCollections = new HashSet<>();
+ for (Violation.ReplicaInfoAndErr replicaInfoAndErr : ctx.violation.getViolatingReplicas()) {
+ uniqueCollections.add(replicaInfoAndErr.replicaInfo.getCollection());
+ }
+
+ collectionLoop:
+ for (String collection : uniqueCollections) {
+ String withCollection = withCollectionsMap.get(collection);
+ if (withCollection == null) continue;
+
+ // can we find a node from which we can move a replica of the `withCollection`
+ // without creating another violation?
+ for (Row row : ctx.session.matrix) {
+ if (ctx.violation.node.equals(row.node)) continue; // filter the violating node
+
+ Set<String> hostedCollections = new HashSet<>();
+ row.forEachReplica(replicaInfo -> hostedCollections.add(replicaInfo.getCollection()));
+
+ if (hostedCollections.contains(withCollection) && !hostedCollections.contains(collection)) {
+ // find the candidate replicas that we can move
+ List<ReplicaInfo> movableReplicas = new ArrayList<>();
+ row.forEachReplica(replicaInfo -> {
+ if (replicaInfo.getCollection().equals(withCollection)) {
+ movableReplicas.add(replicaInfo);
+ }
+ });
+
+ for (ReplicaInfo toMove : movableReplicas) {
+ // candidate source node for a move replica operation
+ Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .forceOperation(true)
+ .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
+ .hint(Suggester.Hint.SRC_NODE, row.node)
+ .hint(Suggester.Hint.REPLICA, toMove.getName())
+ .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
+ if (ctx.addSuggestion(suggester) != null)
+ continue collectionLoop; // one suggestion is enough for this collection
+ }
+ }
+ }
+
+ // we could not find a valid move, so we suggest adding a replica
+ Suggester suggester = ctx.session.getSuggester(ADDREPLICA)
+ .forceOperation(true)
+ .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
+ .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
+ ctx.addSuggestion(suggester);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 2015b52..83fb25a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -36,7 +36,8 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.cloud.autoscaling.Row;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
+import org.apache.solr.client.solrj.cloud.autoscaling.VariableBase;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
import org.apache.solr.common.MapWriter;
@@ -59,9 +60,9 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyMap;
import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.TOTALDISK;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.WITH_COLLECTION;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.TOTALDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.WITH_COLLECTION;
/**
*
@@ -153,7 +154,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
for (String key : keys) {
if (r.getVariables().containsKey(key)) continue;// it's already collected
String perReplicaMetricsKey = "solr.core." + r.getCollection() + "." + r.getShard() + "." + Utils.parseMetricsReplicaName(r.getCollection(), r.getCore()) + ":";
- Suggestion.ConditionType tagType = Suggestion.getTagType(key);
+ Type tagType = VariableBase.getTagType(key);
String perReplicaValue = key;
if (tagType != null) {
perReplicaValue = tagType.metricsAttribute;
@@ -168,7 +169,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
Map<String, Object> tagValues = fetchReplicaMetrics(node, metricsKeyVsTagReplica);
tagValues.forEach((k, o) -> {
Pair<String, ReplicaInfo> p = metricsKeyVsTagReplica.get(k);
- Suggestion.ConditionType validator = Suggestion.getTagType(p.first());
+ Type validator = VariableBase.getTagType(p.first());
if (validator != null) o = validator.convertVal(o);
if (p.second() != null) p.second().getVariables().put(p.first(), o);
});
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 16addd4..16dfdcd 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -43,7 +43,6 @@ import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.DistributedQueueFactory;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.Clause.RangeVal;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
@@ -72,9 +71,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.CORES;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.REPLICA;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.REPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -3413,18 +3412,18 @@ public void testUtilizeNodeFailure2() throws Exception {
" {'node':'solr-27:8983_solr'}]}]}";
List l = (List) ((Map) Utils.fromJSONString(rowsData)).get("sortedNodes");
- List<Suggestion.ConditionType> params = new ArrayList<>();
+ List<Variable.Type> params = new ArrayList<>();
params.add(CORES);
- params.add(Suggestion.ConditionType.FREEDISK);
- params.add(Suggestion.ConditionType.SYSLOADAVG);
- params.add(Suggestion.ConditionType.NODE);
+ params.add(Variable.Type.FREEDISK);
+ params.add(Variable.Type.SYSLOADAVG);
+ params.add(Variable.Type.NODE);
List<Row> rows = new ArrayList<>();
for (Object o : l) {
Map m = (Map) o;
Cell[] c = new Cell[params.size()];
List attrs = (List) m.get("attributes");
for (int i = 0; i < params.size(); i++) {
- Suggestion.ConditionType param = params.get(i);
+ Variable.Type param = params.get(i);
for (Object attr : attrs) {
Object o1 = ((Map) attr).get(param.tagName);
if (o1 != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index 9c5528a..678600f 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -40,7 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyMap;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.CORES;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES;
public class TestPolicy2 extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());