You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/03/20 12:33:04 UTC
[1/2] lucene-solr:feature/autoscaling: SOLR-10278: Implemented
conditions
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling 1fe4cff22 -> cdab4946d
SOLR-10278: Implemented conditions
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a025f8bd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a025f8bd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a025f8bd
Branch: refs/heads/feature/autoscaling
Commit: a025f8bdb93a83853b7f3664d272a0b8aabdd321
Parents: 541469a
Author: Noble Paul <no...@apache.org>
Authored: Mon Mar 20 23:01:03 2017 +1030
Committer: Noble Paul <no...@apache.org>
Committed: Mon Mar 20 23:01:03 2017 +1030
----------------------------------------------------------------------
.../org/apache/solr/common/IteratorWriter.java | 26 +-
.../solr/common/params/CollectionParams.java | 1 +
.../apache/solr/common/util/JavaBinCodec.java | 15 +-
.../java/org/apache/solr/common/util/Utils.java | 22 +-
.../src/java/org/apache/solr/recipe/Clause.java | 145 +++++++++++
.../java/org/apache/solr/recipe/Operand.java | 4 +-
.../java/org/apache/solr/recipe/Preference.java | 62 +++++
.../java/org/apache/solr/recipe/RuleSorter.java | 257 ++++++++++---------
.../org/apache/solr/recipe/TestRuleSorter.java | 102 +++++++-
9 files changed, 471 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
index 15733fb..cbfb584 100644
--- a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
@@ -28,7 +28,7 @@ import java.util.List;
*/
public interface IteratorWriter {
/**
- * @param iw after this method returns , the EntryWriter Object is invalid
+ * @param iw after this method returns , the ItemWriter Object is invalid
* Do not hold a reference to this object
*/
void writeIter(ItemWriter iw) throws IOException;
@@ -65,16 +65,20 @@ public interface IteratorWriter {
return this;
}
}
- default List toList( List l) throws IOException {
- writeIter(new IteratorWriter.ItemWriter() {
- @Override
- public IteratorWriter.ItemWriter add(Object o) throws IOException {
- if (o instanceof MapWriter) o = ((MapWriter) o).toMap(new LinkedHashMap<>());
- if (o instanceof IteratorWriter) o = ((IteratorWriter) o).toList(new ArrayList<>());
- l.add(o);
- return this;
- }
- });
+ default List toList( List l) {
+ try {
+ writeIter(new ItemWriter() {
+ @Override
+ public ItemWriter add(Object o) throws IOException {
+ if (o instanceof MapWriter) o = ((MapWriter) o).toMap(new LinkedHashMap<>());
+ if (o instanceof IteratorWriter) o = ((IteratorWriter) o).toList(new ArrayList<>());
+ l.add(o);
+ return this;
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
return l;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index f1e5a52..dc9efbe 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -94,6 +94,7 @@ public interface CollectionParams {
CREATESNAPSHOT(true, LockLevel.COLLECTION),
DELETESNAPSHOT(true, LockLevel.COLLECTION),
LISTSNAPSHOTS(false, LockLevel.NONE),
+ MOVEREPLICA(false, LockLevel.SHARD),
//only for testing. it just waits for specified time
// these are not exposed via collection API commands
// but the overseer is aware of these tasks
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
index def3571..d9843e1 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
@@ -33,6 +33,8 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.IteratorWriter;
@@ -390,7 +392,18 @@ public class JavaBinCodec implements PushWriter {
writeMap(((MapSerializable) val).toMap(new NamedList().asShallowMap()));
return true;
}
-
+ if (val instanceof AtomicInteger) {
+ writeInt(((AtomicInteger) val).get());
+ return true;
+ }
+ if (val instanceof AtomicLong) {
+ writeLong(((AtomicLong) val).get());
+ return true;
+ }
+ if (val instanceof AtomicBoolean) {
+ writeBoolean(((AtomicBoolean) val).get());
+ return true;
+ }
return false;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 4cb6b8e..ed3f68d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -36,6 +36,8 @@ import java.util.regex.Pattern;
import org.apache.http.HttpEntity;
import org.apache.http.util.EntityUtils;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.noggit.CharArr;
import org.noggit.JSONParser;
@@ -60,23 +62,23 @@ public class Utils {
Map copy = new LinkedHashMap();
for (Object o : map.entrySet()) {
Map.Entry e = (Map.Entry) o;
- Object v = e.getValue();
- if (v instanceof Map) v = getDeepCopy((Map) v, maxDepth - 1, mutable);
- else if (v instanceof Collection) v = getDeepCopy((Collection) v, maxDepth - 1, mutable);
- copy.put(e.getKey(), v);
+ copy.put(e.getKey(), makeDeepCopy(e.getValue(),maxDepth, mutable));
}
return mutable ? copy : Collections.unmodifiableMap(copy);
}
+ private static Object makeDeepCopy(Object v, int maxDepth, boolean mutable) {
+ if (v instanceof MapWriter) v = ((MapWriter) v).toMap(new LinkedHashMap<>());
+ else if (v instanceof IteratorWriter) v = ((IteratorWriter) v).toList(new ArrayList<>());
+ else if (v instanceof Map) v = getDeepCopy((Map) v, maxDepth - 1, mutable);
+ else if (v instanceof Collection) v = getDeepCopy((Collection) v, maxDepth - 1, mutable);
+ return v;
+ }
+
public static Collection getDeepCopy(Collection c, int maxDepth, boolean mutable) {
if (c == null || maxDepth < 1) return c;
Collection result = c instanceof Set ? new HashSet() : new ArrayList();
- for (Object o : c) {
- if (o instanceof Map) {
- o = getDeepCopy((Map) o, maxDepth - 1, mutable);
- }
- result.add(o);
- }
+ for (Object o : c) result.add(makeDeepCopy(o, maxDepth, mutable));
return mutable ? result : result instanceof Set ? unmodifiableSet((Set) result) : unmodifiableList((List) result);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/recipe/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/recipe/Clause.java b/solr/solrj/src/java/org/apache/solr/recipe/Clause.java
new file mode 100644
index 0000000..2bb4fbc
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/recipe/Clause.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.recipe;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.recipe.RuleSorter.Row;
+
+import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
+import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
+import static org.apache.solr.common.params.CoreAdminParams.SHARD;
+import static org.apache.solr.recipe.Operand.EQUAL;
+import static org.apache.solr.recipe.Operand.GREATER_THAN;
+import static org.apache.solr.recipe.Operand.LESS_THAN;
+import static org.apache.solr.recipe.Operand.NOT_EQUAL;
+import static org.apache.solr.recipe.RuleSorter.ANY;
+
+
+class Clause implements MapWriter {
+ Map<String, Object> original;
+ Condition collection, shard, replica, tag;
+ boolean strict = true;
+
+ Clause(Map<String, Object> m) {
+ this.original = m;
+ collection = new Condition(COLLECTION, m.containsKey(COLLECTION) ? (String) m.get(COLLECTION) : ANY, EQUAL);
+ shard = new Condition(SHARD, m.containsKey(SHARD) ? (String) m.get(SHARD) : ANY, EQUAL);
+ String replica = m.containsKey(REPLICA) ? String.valueOf(m.get(REPLICA)) : ANY;
+ this.replica = parse(REPLICA, replica);
+ strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true")));
+ m.forEach(this::parseCondition);
+ if (tag == null)
+ throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
+ }
+
+ void parseCondition(String s, Object o) {
+ if (IGNORE_TAGS.contains(s)) return;
+ if (tag != null) {
+ throw new IllegalArgumentException("Only one tag other than collection, shard, replica is possible");
+ }
+ tag = parse(s, o);
+ }
+
+ class Condition {
+ final String name;
+ final Object val;
+ final Operand op;
+
+ Condition(String name, Object val, Operand op) {
+ this.name = name;
+ this.val = val;
+ this.op = op;
+ }
+
+ boolean isMatch(Object inputVal) {
+ return op.canMatch(val, inputVal);
+ }
+
+ }
+
+ Condition parse(String s, Object o) {
+ Object expectedVal;
+ String value = null;
+ try {
+ String conditionName = s.trim();
+ value = String.valueOf(o).trim();
+ Operand operand = null;
+ if ((expectedVal = NOT_EQUAL.match(value)) != null) {
+ operand = NOT_EQUAL;
+ } else if ((expectedVal = GREATER_THAN.match(value)) != null) {
+ operand = GREATER_THAN;
+ } else if ((expectedVal = LESS_THAN.match(value)) != null) {
+ operand = LESS_THAN;
+ } else {
+ operand = EQUAL;
+ expectedVal = EQUAL.match(value);
+ }
+
+ return new Condition(conditionName, expectedVal, operand);
+
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid tag : " + s + ":" + value, e);
+ }
+ }
+
+
+ TestStatus test(Row row) {
+ AtomicReference<TestStatus> result = new AtomicReference<>(TestStatus.NOT_APPLICABLE);
+ Object val = row.getVal(tag.name);
+ if (tag.isMatch(val)) {
+ checkReplicaCount(row, result);
+ if (result.get() == TestStatus.FAIL) row.violations.add(this);
+ }
+ return result.get();
+ }
+
+ TestStatus checkReplicaCount(Row row, AtomicReference<TestStatus> result) {
+ row.replicaInfo.forEach((coll, e) -> {
+ if (!collection.isMatch(coll)) return;
+ AtomicInteger count = new AtomicInteger();
+ e.forEach((sh, replicas) -> {
+ if (!shard.isMatch(sh)) return;
+ count.addAndGet(replicas.size());
+ });
+ result.set(replica.isMatch(count) ? TestStatus.PASS : TestStatus.FAIL);
+ if (RuleSorter.EACH.equals(shard.val)) count.set(0);
+ });
+ return TestStatus.PASS;
+ }
+
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (Map.Entry<String, Object> e : original.entrySet()) ew.put(e.getKey(), e.getValue());
+ }
+
+ enum TestStatus {
+ NOT_APPLICABLE, FAIL, PASS
+ }
+
+ private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict"));
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/recipe/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/recipe/Operand.java b/solr/solrj/src/java/org/apache/solr/recipe/Operand.java
index f755dad..1ef82af 100644
--- a/solr/solrj/src/java/org/apache/solr/recipe/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/recipe/Operand.java
@@ -19,9 +19,7 @@ package org.apache.solr.recipe;
import java.util.Objects;
-/**
- * Created by noble on 3/6/17.
- */
+
public enum Operand {
EQUAL(""),
NOT_EQUAL("!") {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/recipe/Preference.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/recipe/Preference.java b/solr/solrj/src/java/org/apache/solr/recipe/Preference.java
new file mode 100644
index 0000000..9d8e085
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/recipe/Preference.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.recipe;
+
+import java.util.List;
+import java.util.Map;
+
+class Preference {
+ final RuleSorter.SortParam name;
+ Integer precision;
+ final RuleSorter.Sort sort;
+ Preference next;
+ public int idx;
+
+ Preference(Map<String, Object> m) {
+ sort = RuleSorter.Sort.get(m);
+ name = RuleSorter.SortParam.get(m.get(sort.name()).toString());
+ Object p = m.getOrDefault("precision", 0);
+ precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
+
+ }
+
+ // there are 2 modes of compare.
+ // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
+ // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
+ int compare(RuleSorter.Row r1, RuleSorter.Row r2, boolean recursive) {
+ Object o1 = recursive ? r1.cells[idx].val_ : r1.cells[idx].val;
+ Object o2 = recursive ? r2.cells[idx].val_ : r2.cells[idx].val;
+ int result = 0;
+ if (o1 instanceof Integer && o2 instanceof Integer) result = ((Integer) o1).compareTo((Integer) o2);
+ if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
+ if (o1 instanceof Float && o2 instanceof Float) result = ((Float) o1).compareTo((Float) o2);
+ if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
+ return result == 0 ? next == null ? 0 : next.compare(r1, r2, recursive) : sort.sortval * result;
+ }
+
+ //sets the new value according to precision in val_
+ void setApproxVal(List<RuleSorter.Row> tmpMatrix) {
+ Object prevVal = null;
+ for (RuleSorter.Row row : tmpMatrix) {
+ prevVal = row.cells[idx].val_ =
+ prevVal == null || Math.abs(((Number) prevVal).longValue() - ((Number) row.cells[idx].val).longValue()) > precision ?
+ row.cells[idx].val :
+ prevVal;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java b/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java
index 1ed6800..b19696d 100644
--- a/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java
+++ b/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java
@@ -20,34 +20,40 @@ package org.apache.solr.recipe;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.util.Utils;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
-import static org.apache.solr.recipe.Operand.GREATER_THAN;
-import static org.apache.solr.recipe.Operand.LESS_THAN;
-import static org.apache.solr.recipe.Operand.NOT_EQUAL;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
+import static org.apache.solr.common.params.CoreAdminParams.NODE;
public class RuleSorter {
- public static final String ALL = "#ALL";
public static final String EACH = "#EACH";
- List<Clause> conditionClauses = new ArrayList<>();
+ public static final String ANY = "#ANY";
+ List<Clause> clauses = new ArrayList<>();
List<Preference> preferences = new ArrayList<>();
List<String> params= new ArrayList<>();
public RuleSorter(Map<String, Object> jsonMap) {
List<Map<String, Object>> l = getListOfMap("conditions", jsonMap);
- conditionClauses = l.stream().map(Clause::new).collect(toList());
+ clauses = l.stream().map(Clause::new).collect(toList());
l = getListOfMap("preferences", jsonMap);
preferences = l.stream().map(Preference::new).collect(toList());
for (int i = 0; i < preferences.size() - 1; i++) {
@@ -55,10 +61,7 @@ public class RuleSorter {
preference.next = preferences.get(i + 1);
}
- for (Clause c : conditionClauses) {
- for (Condition condition : c.conditions) params.add(condition.name);
- }
-
+ for (Clause c : clauses) params.add(c.tag.name);
for (Preference preference : preferences) {
if (params.contains(preference.name.name())) {
throw new RuntimeException(preference.name + " is repeated");
@@ -66,34 +69,64 @@ public class RuleSorter {
params.add(preference.name.toString());
preference.idx = params.size() - 1;
}
-
}
+
public class Session implements MapWriter {
- private final List<String> nodes;
- private final NodeValueProvider snitch;
- private List<Row> matrix;
- List<String> paramsList = new ArrayList<>(params);
+ final List<String> nodes;
+ final NodeValueProvider snitch;
+ final List<Row> matrix;
+ Set<String> collections = new HashSet<>();
- private Session(List<String> nodes, NodeValueProvider snitch) {
+ Session(List<String> nodes, NodeValueProvider snitch) {
this.nodes = nodes;
this.snitch = snitch;
matrix = new ArrayList<>(nodes.size());
- for (String node : nodes) matrix.add(new Row(node, paramsList, snitch));
+ for (String node : nodes) matrix.add(new Row(node, params, snitch));
+ for (Row row : matrix) row.replicaInfo.forEach((s, e) -> collections.add(s));
+ }
+ List<Row> getMatrixCopy() {
+ return matrix.stream()
+ .map(Row::copy)
+ .collect(Collectors.toList());
}
- public void sort() {
- if (preferences.size() > 1) {
+
+ /**Apply the preferences and conditions
+ */
+ public void applyRules() {
+ if (!preferences.isEmpty()) {
//this is to set the approximate value according to the precision
ArrayList<Row> tmpMatrix = new ArrayList<>(matrix);
for (Preference p : preferences) {
Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false));
- p.setNewVal(tmpMatrix);
+ p.setApproxVal(tmpMatrix);
}
//approximate values are set now. Let's do recursive sorting
Collections.sort(matrix, (r1, r2) -> preferences.get(0).compare(r1, r2, true));
}
+
+ if (!clauses.isEmpty()) {
+ for (Clause clause : clauses) {
+ for (Row row : matrix) {
+ clause.test(row);
+ }
+ }
+ }
+
+ }
+
+ public Map<String, List<Clause>> getViolations() {
+ return matrix.stream()
+ .filter(row -> !row.violations.isEmpty())
+ .collect(Collectors.toMap(r -> r.node, r -> r.violations));
+ }
+
+ public Operation suggest(CollectionAction action) {
+ if (!supportedActions.contains(action))
+ throw new UnsupportedOperationException(action.toString() + "is not supported");
+ return null;
}
@Override
@@ -130,103 +163,6 @@ public class RuleSorter {
}
- static class Clause {
- List<Condition> conditions;
- boolean strict = true;
-
- Clause(Map<String, Object> m) {
- conditions = m.entrySet().stream()
- .filter(e -> !"strict".equals(e.getKey().trim()))
- .map(Condition::new)
- .collect(toList());
- Object o = m.get("strict");
- if (o == null) return;
- strict = o instanceof Boolean ? (Boolean) o : Boolean.parseBoolean(o.toString());
- }
-
- }
-
- static class Condition {
- String name;
- Object val;
- Operand operand;
-
- Condition(Map.Entry<String, Object> m) {
- Object expectedVal;
- try {
- this.name = m.getKey().trim();
- String value = m.getValue().toString().trim();
- if ((expectedVal = NOT_EQUAL.match(value)) != null) {
- operand = NOT_EQUAL;
- } else if ((expectedVal = GREATER_THAN.match(value)) != null) {
- operand = GREATER_THAN;
- } else if ((expectedVal = LESS_THAN.match(value)) != null) {
- operand = LESS_THAN;
- } else {
- operand = Operand.EQUAL;
- expectedVal = value;
- }
-
- if (name.equals(REPLICA_PROP)) {
- if (!ALL.equals(expectedVal)) {
- try {
- expectedVal = Integer.parseInt(expectedVal.toString());
- } catch (NumberFormatException e) {
- throw new RuntimeException("The replica tag value can only be '*' or an integer");
- }
- }
- }
-
- } catch (Exception e) {
- throw new IllegalArgumentException("Invalid condition : " + name + ":" + val, e);
- }
- this.val = expectedVal;
-
-
- }
- }
-
- static class Preference {
- final SortParam name;
- Integer precision;
- final Sort sort;
- Preference next;
- public int idx;
-
- Preference(Map<String, Object> m) {
- sort = Sort.get(m);
- name = SortParam.get(m.get(sort.name()).toString());
- Object p = m.getOrDefault("precision", 0);
- precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
-
- }
-
- // there are 2 modes of compare.
- // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
- // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
- int compare(Row r1, Row r2, boolean recursive) {
- Object o1 = recursive ? r1.cells[idx].val_ : r1.cells[idx].val;
- Object o2 = recursive ? r2.cells[idx].val_ : r2.cells[idx].val;
- int result = 0;
- if (o1 instanceof Integer && o2 instanceof Integer) result = ((Integer) o1).compareTo((Integer) o2);
- if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
- if (o1 instanceof Float && o2 instanceof Float) result = ((Float) o1).compareTo((Float) o2);
- if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
- return result == 0 ? next == null ? 0 : next.compare(r1, r2, recursive) : sort.sortval * result;
- }
-
- //sets the new value according to precision in val_
- void setNewVal(List<Row> tmpMatrix) {
- Object prevVal = null;
- for (Row row : tmpMatrix) {
- prevVal = row.cells[idx].val_ =
- prevVal == null || Math.abs(((Number) prevVal).longValue() - ((Number) row.cells[idx].val).longValue()) > precision ?
- row.cells[idx].val :
- prevVal;
- }
- }
- }
-
enum SortParam {
freedisk, cores, heap, cpu;
@@ -245,7 +181,7 @@ public class RuleSorter {
sortval = i;
}
- public static Sort get(Map<String, Object> m) {
+ static Sort get(Map<String, Object> m) {
if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) {
throw new RuntimeException("Cannot have both 'maximize' and 'minimize'");
}
@@ -255,30 +191,54 @@ public class RuleSorter {
}
}
- public static class Row implements MapWriter {
+ static class Row implements MapWriter {
public final String node;
final Cell[] cells;
+ Map<String, Map<String, List<ReplicaStat>>> replicaInfo;
+ List<Clause> violations = new ArrayList<>();
boolean anyValueMissing = false;
Row(String node, List<String> params, NodeValueProvider snitch) {
+ replicaInfo = snitch.getReplicaCounts(node, params);
+ if (replicaInfo == null) replicaInfo = Collections.emptyMap();
this.node = node;
cells = new Cell[params.size()];
- Map<String, Object> vals = new HashMap<>();
- for (String param : params) vals.put(param, null);
- snitch.getValues(node, vals);
+ Map<String, Object> vals = snitch.getValues(node, params);
for (int i = 0; i < params.size(); i++) {
String s = params.get(i);
cells[i] = new Cell(i, s, vals.get(s));
+ if (NODE.equals(s)) cells[i].val = node;
if (cells[i].val == null) anyValueMissing = true;
}
}
+ Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaStat>>> replicaInfo) {
+ this.node = node;
+ this.cells = new Cell[cells.length];
+ for (int i = 0; i < this.cells.length; i++) {
+ this.cells[i] = cells[i].copy();
+
+ }
+ this.anyValueMissing = anyValueMissing;
+ this.replicaInfo = replicaInfo;
+ }
+
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put(node, (IteratorWriter) iw -> {
+ iw.add((MapWriter) e -> e.put("replicas", replicaInfo));
for (Cell cell : cells) iw.add(cell);
});
}
+
+ public Row copy() {
+ return new Row(node, cells, anyValueMissing, replicaInfo);
+ }
+
+ public Object getVal(String name) {
+ for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
+ return null;
+ }
}
static class Cell implements MapWriter {
@@ -292,15 +252,60 @@ public class RuleSorter {
this.val = val;
}
+ Cell(int index, String name, Object val, Object val_) {
+ this.index = index;
+ this.name = name;
+ this.val = val;
+ this.val_ = val_;
+ }
+
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put(name, val);
}
+
+ public Cell copy() {
+ return new Cell(index, name, val, val_);
+ }
}
- interface NodeValueProvider {
+ static class Operation {
+ CollectionAction action;
+ String node, collection, shard, replica;
+ String targetNode;
+
+ }
+
+
+ static class ReplicaStat implements MapWriter {
+ final String name;
+ Map<String, Object> variables;
+
+ ReplicaStat(String name, Map<String, Object> vals) {
+ this.name = name;
+ this.variables = vals;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, variables);
+ }
+ }
- void getValues(String node, Map<String, Object> valuesMap);
+ interface NodeValueProvider {
+ Map<String, Object> getValues(String node, Collection<String> keys);
+
+ /**
+ * Get the details of each replica in a node. It attempts to fetch as much details about
+ * the replica as mentioned in the keys list
+ * <p>
+ * the format is {collection:shard :[{replicaetails}]}
+ */
+ Map<String, Map<String, List<ReplicaStat>>> getReplicaCounts(String node, Collection<String> keys);
}
+
+ private static final Set<CollectionAction> supportedActions = new HashSet<>(Arrays.asList(ADDREPLICA, DELETEREPLICA, MOVEREPLICA, SPLITSHARD));
+
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a025f8bd/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java b/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java
index e72aafb..2f5ae90 100644
--- a/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java
+++ b/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java
@@ -18,18 +18,23 @@
package org.apache.solr.recipe;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.common.util.ValidatingJsonMap;
public class TestRuleSorter extends SolrTestCaseJ4 {
- public void testRuleParsing() {
-
+ public void testRuleParsing() throws IOException {
String rules = "{" +
- "conditions:[{node:'!overseer', strict:false}, " +
+ "conditions:[{nodeRole:'!overseer', strict:false}, " +
"{replica:'<2',node:'*', shard:'#EACH'}]," +
" preferences:[" +
"{minimize:cores , precision:2}," +
@@ -37,19 +42,93 @@ public class TestRuleSorter extends SolrTestCaseJ4 {
"{minimize:heap, precision:1000}]}";
- Map nodeValues = (Map) Utils.fromJSONString( "{" +
+ Map<String,Map> nodeValues = (Map<String, Map>) Utils.fromJSONString( "{" +
"node1:{cores:12, freedisk: 334, heap:10480}," +
"node2:{cores:4, freedisk: 749, heap:6873}," +
"node3:{cores:7, freedisk: 262, heap:7834}," +
"node4:{cores:8, freedisk: 375, heap:16900}" +
"}");
+ String clusterState = "{'gettingstarted':{" +
+ " 'router':{'name':'compositeId'}," +
+ " 'shards':{" +
+ " 'shard1':{" +
+ " 'range':'80000000-ffffffff'," +
+ " 'state':'active'," +
+ " 'replicas':{" +
+ " 'core_node1':{" +
+ " 'core':'gettingstarted_shard1_replica1'," +
+ " 'base_url':'http://10.0.0.4:8983/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'," +
+ " 'leader':'true'}," +
+ " 'core_node4':{" +
+ " 'core':'gettingstarted_shard1_replica2'," +
+ " 'base_url':'http://10.0.0.4:7574/solr'," +
+ " 'node_name':'node2'," +
+ " 'state':'active'}}}," +
+ " 'shard2':{" +
+ " 'range':'0-7fffffff'," +
+ " 'state':'active'," +
+ " 'replicas':{" +
+ " 'core_node2':{" +
+ " 'core':'gettingstarted_shard2_replica1'," +
+ " 'base_url':'http://10.0.0.4:8983/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'," +
+ " 'leader':'true'}," +
+ " 'core_node3':{" +
+ " 'core':'gettingstarted_shard2_replica2'," +
+ " 'base_url':'http://10.0.0.4:7574/solr'," +
+ " 'node_name':'node2'," +
+ " 'state':'active'}}}}}}";
+
+
+ ValidatingJsonMap m = ValidatingJsonMap
+ .getDeepCopy((Map) Utils.fromJSONString(clusterState), 6, true);
+
RuleSorter ruleSorter = new RuleSorter((Map<String, Object>) Utils.fromJSONString(rules));
- RuleSorter.Session session = ruleSorter.createSession(Arrays.asList("node1", "node2","node3","node4"), (node, valuesMap) -> {
- Map n = (Map) nodeValues.get(node);
- valuesMap.entrySet().stream().forEach(e -> e.setValue(n.get(e.getKey())));
- });
- session.sort();
+ RuleSorter.Session session;
+ RuleSorter.NodeValueProvider snitch = new RuleSorter.NodeValueProvider() {
+ @Override
+ public Map<String,Object> getValues(String node, Collection<String> keys) {
+ Map<String,Object> result = new LinkedHashMap<>();
+ keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+
+ }
+
+ @Override
+ public Map<String, Map<String, List<RuleSorter.ReplicaStat>>> getReplicaCounts(String node, Collection<String> keys) {
+ Map<String, Map<String, List<RuleSorter.ReplicaStat>>> result = new LinkedHashMap<>();
+
+ m.forEach((collName, o) -> {
+ ValidatingJsonMap coll = (ValidatingJsonMap) o;
+ coll.getMap("shards").forEach((shard, o1) -> {
+ ValidatingJsonMap sh = (ValidatingJsonMap) o1;
+ sh.getMap("replicas").forEach((replicaName, o2) -> {
+ ValidatingJsonMap r = (ValidatingJsonMap) o2;
+ String node_name = (String) r.get("node_name");
+ if (!node_name.equals(node)) return;
+ Map<String, List<RuleSorter.ReplicaStat>> shardVsReplicaStats = result.get(collName);
+ if (shardVsReplicaStats == null) result.put(collName, shardVsReplicaStats = new HashMap<>());
+ List<RuleSorter.ReplicaStat> replicaStats = shardVsReplicaStats.get(shard);
+ if (replicaStats == null) shardVsReplicaStats.put(shard, replicaStats = new ArrayList<>());
+ replicaStats.add(new RuleSorter.ReplicaStat(replicaName, new HashMap<>()));
+
+ });
+ });
+ });
+
+ return result;
+ }
+
+
+ };
+
+ session = ruleSorter.createSession(Arrays.asList("node1", "node2", "node3", "node4"), snitch);
+
+ session.applyRules();
List<RuleSorter.Row> l = session.getSorted();
assertEquals("node1",l.get(0).node);
assertEquals("node3",l.get(1).node);
@@ -57,11 +136,10 @@ public class TestRuleSorter extends SolrTestCaseJ4 {
assertEquals("node2",l.get(3).node);
-// System.out.println(session);
+ System.out.printf(Utils.toJSONString(Utils.getDeepCopy(session.toMap(new LinkedHashMap<>()), 8)));
+ System.out.println(Utils.getDeepCopy(session.getViolations(), 6));
}
-
-
}
[2/2] lucene-solr:feature/autoscaling: Merge branch
'feature/autoscaling' of https://git-wip-us.apache.org/repos/asf/lucene-solr
into feature/autoscaling
Posted by no...@apache.org.
Merge branch 'feature/autoscaling' of https://git-wip-us.apache.org/repos/asf/lucene-solr into feature/autoscaling
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cdab4946
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cdab4946
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cdab4946
Branch: refs/heads/feature/autoscaling
Commit: cdab4946db1d99f3ea4f7697c6d70ddaa359bf50
Parents: a025f8b 1fe4cff
Author: Noble Paul <no...@apache.org>
Authored: Mon Mar 20 23:01:51 2017 +1030
Committer: Noble Paul <no...@apache.org>
Committed: Mon Mar 20 23:01:51 2017 +1030
----------------------------------------------------------------------
lucene/CHANGES.txt | 7 +
.../byTask/feeds/EnwikiContentSource.java | 2 +-
.../benchmark/byTask/tasks/ForceMergeTask.java | 2 +-
.../org/apache/lucene/util/fst/TestFSTs.java | 4 +-
.../lucene/search/join/TestBlockJoin.java | 2 +-
.../search/TestDiversifiedTopDocsCollector.java | 2 +-
.../queries/function/TestValueSources.java | 6 +-
.../queryparser/classic/QueryParserBase.java | 6 +-
.../standard/parser/StandardSyntaxParser.java | 8 +-
.../standard/parser/StandardSyntaxParser.jj | 8 +-
.../surround/parser/QueryParser.java | 2 +-
.../queryparser/surround/parser/QueryParser.jj | 2 +-
.../xml/builders/PointRangeQueryBuilder.java | 16 +-
.../queryparser/classic/TestQueryParser.java | 2 +-
.../xml/CoreParserTestIndexData.java | 2 +-
.../lucene/document/LatLonDocValuesField.java | 2 +-
.../lucene/spatial3d/Geo3DDocValuesField.java | 2 +-
solr/CHANGES.txt | 21 +
solr/bin/solr | 57 ++-
solr/bin/solr.in.sh | 11 +-
.../analytics/util/RangeEndpointCalculator.java | 8 +-
.../handler/dataimport/MailEntityProcessor.java | 2 +-
.../TikaLanguageIdentifierUpdateProcessor.java | 5 +-
.../solr/hadoop/TreeMergeOutputFormat.java | 3 +-
.../java/org/apache/solr/response/PageTool.java | 2 +-
.../org/apache/solr/cloud/ElectionContext.java | 20 +-
.../OverseerAutoReplicaFailoverThread.java | 6 +-
.../cloud/OverseerCollectionMessageHandler.java | 1 +
.../org/apache/solr/cloud/RecoveryStrategy.java | 29 +-
.../apache/solr/cloud/ReplicateFromLeader.java | 124 ++++++
.../org/apache/solr/cloud/ZkController.java | 39 +-
.../org/apache/solr/core/CoreContainer.java | 30 +-
.../org/apache/solr/core/RequestParams.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 43 +-
.../org/apache/solr/handler/IndexFetcher.java | 40 +-
.../apache/solr/handler/ReplicationHandler.java | 30 +-
.../org/apache/solr/handler/SQLHandler.java | 37 +-
.../solr/handler/admin/CollectionsHandler.java | 4 +-
.../solr/handler/admin/MetricsHandler.java | 6 +-
.../handler/admin/PropertiesRequestHandler.java | 27 +-
.../solr/handler/admin/SystemInfoHandler.java | 20 +-
.../handler/component/RangeFacetRequest.java | 8 +-
.../solr/handler/component/SearchHandler.java | 5 +
.../apache/solr/handler/sql/SolrEnumerator.java | 4 +-
.../org/apache/solr/handler/sql/SolrTable.java | 10 +-
.../UninvertDocValuesMergePolicyFactory.java | 218 ++++++++++
.../apache/solr/metrics/SolrMetricManager.java | 5 +-
.../reporters/solr/SolrClusterReporter.java | 10 +-
.../metrics/reporters/solr/SolrReporter.java | 2 +-
.../reporters/solr/SolrShardReporter.java | 5 +-
.../apache/solr/parser/SolrQueryParserBase.java | 4 +-
.../apache/solr/schema/NumericFieldType.java | 4 +-
.../java/org/apache/solr/search/Grouping.java | 2 +-
.../apache/solr/search/SolrIndexSearcher.java | 2 +-
.../apache/solr/search/facet/FacetRange.java | 8 +-
.../SearchGroupShardResponseProcessor.java | 2 +-
.../security/AutorizationEditOperation.java | 2 +-
.../org/apache/solr/update/CommitTracker.java | 5 +
.../solr/update/DirectUpdateHandler2.java | 53 ++-
.../apache/solr/update/HdfsTransactionLog.java | 50 +++
.../apache/solr/update/SolrIndexSplitter.java | 3 +-
.../org/apache/solr/update/SolrIndexWriter.java | 31 +-
.../org/apache/solr/update/TransactionLog.java | 50 +++
.../org/apache/solr/update/UpdateCommand.java | 1 +
.../java/org/apache/solr/update/UpdateLog.java | 185 +++++++-
.../processor/DistributedUpdateProcessor.java | 38 +-
.../TolerantUpdateProcessorFactory.java | 2 +-
.../org/apache/solr/util/DateMathParser.java | 2 +-
.../org/apache/solr/util/RedactionUtils.java | 51 +++
.../org/apache/solr/util/SolrPluginUtils.java | 4 +-
.../org/apache/solr/util/TestInjection.java | 54 +++
...entedPoolingHttpClientConnectionManager.java | 38 +-
.../org/apache/solr/util/stats/MetricUtils.java | 76 ++--
.../solr/collection1/conf/schema-docValues.xml | 1 +
...nfig-uninvertdocvaluesmergepolicyfactory.xml | 38 ++
.../conf/schema.xml | 31 ++
.../conf/solrconfig.xml | 48 ++
.../solr/cloud/BasicDistributedZk2Test.java | 6 +
.../solr/cloud/BasicDistributedZkTest.java | 9 +-
.../cloud/ChaosMonkeyNothingIsSafeTest.java | 7 +
.../org/apache/solr/cloud/ForceLeaderTest.java | 6 +
.../apache/solr/cloud/HttpPartitionTest.java | 7 +
.../LeaderInitiatedRecoveryOnCommitTest.java | 7 +
.../solr/cloud/OnlyLeaderIndexesTest.java | 435 +++++++++++++++++++
.../solr/cloud/RecoveryAfterSoftCommitTest.java | 7 +-
.../org/apache/solr/cloud/ShardSplitTest.java | 6 +
.../cloud/SharedFSAutoReplicaFailoverTest.java | 29 ++
.../apache/solr/cloud/TestCloudRecovery.java | 16 +-
.../apache/solr/cloud/TestCollectionAPI.java | 6 +-
.../cloud/hdfs/HdfsBasicDistributedZkTest.java | 7 +-
.../core/ConfigureRecoveryStrategyTest.java | 1 +
.../core/snapshots/TestSolrCloudSnapshots.java | 2 +-
.../core/snapshots/TestSolrCoreSnapshots.java | 2 +-
.../org/apache/solr/handler/TestSQLHandler.java | 31 ++
.../solr/handler/admin/MetricsHandlerTest.java | 22 +-
.../admin/PropertiesRequestHandlerTest.java | 73 ++++
.../index/UninvertDocValuesMergePolicyTest.java | 243 +++++++++++
.../org/apache/solr/schema/TestPointFields.java | 161 ++++---
.../solr/search/TestCollapseQParserPlugin.java | 2 +-
.../solr/search/TestSolrFieldCacheMBean.java | 4 +-
.../solr/search/mlt/CloudMLTQParserTest.java | 14 +-
.../solr/update/DirectUpdateHandlerTest.java | 18 +-
.../solr/update/SolrIndexMetricsTest.java | 5 +-
.../solr/update/TestInPlaceUpdatesDistrib.java | 25 +-
.../apache/solr/util/stats/MetricUtilsTest.java | 53 ++-
solr/server/scripts/cloud-scripts/zkcli.bat | 2 +-
solr/server/scripts/cloud-scripts/zkcli.sh | 2 +-
.../conf/solrconfig.xml | 2 +-
.../org/apache/solr/client/solrj/io/Tuple.java | 21 +-
.../client/solrj/io/eval/EqualsEvaluator.java | 2 +-
.../solrj/request/CollectionAdminRequest.java | 6 +
.../apache/solr/common/cloud/DocCollection.java | 14 +-
.../apache/solr/common/cloud/ZkStateReader.java | 1 +
.../solr/common/params/SolrParamTest.java | 4 +-
.../cloud/AbstractFullDistribZkTestBase.java | 14 +-
solr/webapp/web/WEB-INF/web.xml | 2 +-
116 files changed, 2560 insertions(+), 408 deletions(-)
----------------------------------------------------------------------