You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/06/28 06:51:40 UTC
[01/18] lucene-solr:feature/autoscaling: SOLR-10307: Export variables
so that solr.in.sh works correctly.
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling 257ba055d -> 39c6fb2e3
SOLR-10307: Export variables so that solr.in.sh works correctly.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/54fc1ee4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/54fc1ee4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/54fc1ee4
Branch: refs/heads/feature/autoscaling
Commit: 54fc1ee44d32ad6149aaa0d13cfc6246b5a7f06d
Parents: 5d44243
Author: Mark Miller <ma...@apache.org>
Authored: Tue Jun 27 02:37:15 2017 -0400
Committer: Mark Miller <ma...@apache.org>
Committed: Tue Jun 27 02:37:15 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +-
solr/bin/solr | 13 ++++++++++++-
2 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/54fc1ee4/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8d5c321..7720ef2 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -373,7 +373,7 @@ New Features
* SOLR-10849: MoreLikeThisComponent should expose setMaxDocFreqPct (maxDoc
frequency percentage). (Dawid Weiss)
-* SOLR-10307: Allow Passing SSL passwords through environment variables. (Mano Kovacs via Mark Miller)
+* SOLR-10307: Allow Passing SSL passwords through environment variables. (Mano Kovacs, Michael Suzuki via Mark Miller)
* SOLR-10721: Provide a way to know when Core Discovery is finished and when all async cores are done loading
(Erick Erickson)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/54fc1ee4/solr/bin/solr
----------------------------------------------------------------------
diff --git a/solr/bin/solr b/solr/bin/solr
index 14f24eb..cd27fda 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -176,7 +176,9 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
if [ -n "$SOLR_SSL_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE"
fi
-
+ if [ -n "$SOLR_SSL_KEY_STORE_PASSWORD" ]; then
+ export SOLR_SSL_KEY_STORE_PASSWORD=$SOLR_SSL_KEY_STORE_PASSWORD
+ fi
if [ -n "$SOLR_SSL_KEY_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore.type=$SOLR_SSL_KEY_STORE_TYPE"
fi
@@ -184,6 +186,9 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
if [ -n "$SOLR_SSL_TRUST_STORE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore=$SOLR_SSL_TRUST_STORE"
fi
+ if [ -n "$SOLR_SSL_TRUST_STORE_PASSWORD" ]; then
+ export SOLR_SSL_TRUST_STORE_PASSWORD=$SOLR_SSL_TRUST_STORE_PASSWORD
+ fi
if [ -n "$SOLR_SSL_TRUST_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore.type=$SOLR_SSL_TRUST_STORE_TYPE"
fi
@@ -198,6 +203,9 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
if [ -n "$SOLR_SSL_CLIENT_KEY_STORE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_CLIENT_KEY_STORE"
+ if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD" ]; then
+ export SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD
+ fi
if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreType=$SOLR_SSL_CLIENT_KEY_STORE_TYPE"
fi
@@ -213,6 +221,9 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStore=$SOLR_SSL_CLIENT_TRUST_STORE"
+ if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" ]; then
+ export SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD
+ fi
if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE_TYPE" ]; then
SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStoreType=$SOLR_SSL_CLIENT_TRUST_STORE_TYPE"
fi
[12/18] lucene-solr:feature/autoscaling: SOLR-10931: Refactoring the
package name for autoscaling client classes
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
deleted file mode 100644
index 5e4078a..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
+++ /dev/null
@@ -1,460 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.Utils;
-
-import static java.util.Collections.singletonMap;
-import static org.apache.solr.cloud.autoscaling.Clause.TestStatus.PASS;
-import static org.apache.solr.cloud.autoscaling.Operand.EQUAL;
-import static org.apache.solr.cloud.autoscaling.Operand.GREATER_THAN;
-import static org.apache.solr.cloud.autoscaling.Operand.LESS_THAN;
-import static org.apache.solr.cloud.autoscaling.Operand.NOT_EQUAL;
-import static org.apache.solr.cloud.autoscaling.Operand.WILDCARD;
-import static org.apache.solr.cloud.autoscaling.Policy.ANY;
-import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
-import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
-import static org.apache.solr.common.params.CoreAdminParams.SHARD;
-
-// a set of conditions in a policy
-public class Clause implements MapWriter, Comparable<Clause> {
- Map<String, Object> original;
- Condition collection, shard, replica, tag, globalTag;
-
- boolean strict = true;
-
- Clause(Map<String, Object> m) {
- this.original = m;
- strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true")));
- Optional<String> globalTagName = m.keySet().stream().filter(Policy.GLOBAL_ONLY_TAGS::contains).findFirst();
- if (globalTagName.isPresent()) {
- globalTag = parse(globalTagName.get(), m);
- if (m.size() > 2) {
- throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + Utils.toJSONString(m));
- }
- tag = parse(m.keySet().stream()
- .filter(s -> (!globalTagName.get().equals(s) && !IGNORE_TAGS.contains(s)))
- .findFirst().get(), m);
- } else {
- collection = parse(COLLECTION, m);
- shard = parse(SHARD, m);
- if(m.get(REPLICA) == null){
- throw new RuntimeException(StrUtils.formatString("'replica' is required in {0}", Utils.toJSONString(m)));
- }
- this.replica = parse(REPLICA, m);
- if (replica.op == WILDCARD) throw new RuntimeException("replica val cannot be null" + Utils.toJSONString(m));
- m.forEach((s, o) -> parseCondition(s, o));
- }
- if (tag == null)
- throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
-
- }
-
- public boolean doesOverride(Clause that) {
- return (collection.equals(that.collection) &&
- tag.name.equals(that.tag.name));
-
- }
-
- public boolean isPerCollectiontag() {
- return globalTag == null;
- }
-
- void parseCondition(String s, Object o) {
- if (IGNORE_TAGS.contains(s)) return;
- if (tag != null) {
- throw new IllegalArgumentException("Only one tag other than collection, shard, replica is possible");
- }
- tag = parse(s, singletonMap(s, o));
- }
-
- @Override
- public int compareTo(Clause that) {
- try {
- int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
- if (v != 0) return v;
- if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
- v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
- if (v == 0) {
- v = Long.compare((Long) this.replica.val, (Long) that.replica.val);
- v = this.replica.op == LESS_THAN ? v : v * -1;
- }
- return v;
- } else {
- return 0;
- }
- } catch (NullPointerException e) {
- throw e;
- }
- }
-
- void addTags(List<String> params) {
- if (globalTag != null && !params.contains(globalTag.name)) params.add(globalTag.name);
- if (tag != null && !params.contains(tag.name)) params.add(tag.name);
- }
-
- static class Condition {
- final String name;
- final Object val;
- final Operand op;
-
- Condition(String name, Object val, Operand op) {
- this.name = name;
- this.val = val;
- this.op = op;
- }
-
- TestStatus match(Row row) {
- return op.match(val, row.getVal(name));
- }
-
- TestStatus match(Object testVal) {
- return op.match(this.val, testVal);
- }
-
- boolean isPass(Object inputVal) {
- return op.match(val, validate(name, inputVal, false)) == PASS;
- }
-
- boolean isPass(Row row) {
- return op.match(val, row.getVal(name)) == PASS;
- }
-
- @Override
- public boolean equals(Object that) {
- if (that instanceof Condition) {
- Condition c = (Condition) that;
- return Objects.equals(c.name, name) && Objects.equals(c.val, val) && c.op == op;
- }
- return false;
- }
-
- public Integer delta(Object val) {
- return op.delta(this.val, val);
- }
- }
-
- static Condition parse(String s, Map m) {
- Object expectedVal = null;
- Object val = m.get(s);
- try {
- String conditionName = s.trim();
- Operand operand = null;
- if (val == null) {
- operand = WILDCARD;
- expectedVal = Policy.ANY;
- } else if (val instanceof String) {
- String strVal = ((String) val).trim();
- if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
- else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
- else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
- else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
- else operand = EQUAL;
- expectedVal = validate(s, strVal.substring(EQUAL == operand || WILDCARD == operand ? 0 : 1), true);
- } else if (val instanceof Number) {
- operand = EQUAL;
- expectedVal = validate(s, val, true);
- }
- return new Condition(conditionName, expectedVal, operand);
-
- } catch (Exception e) {
- throw new IllegalArgumentException("Invalid tag : " + s + ":" + val, e);
- }
- }
-
- public class Violation implements MapWriter {
- final String shard, coll, node;
- final Object actualVal;
- final Integer delta;//how far is the actual value from the expected value
- final Object tagKey;
- private final int hash;
-
-
- private Violation(String coll, String shard, String node, Object actualVal, Integer delta, Object tagKey) {
- this.shard = shard;
- this.coll = coll;
- this.node = node;
- this.delta = delta;
- this.actualVal = actualVal;
- this.tagKey = tagKey;
- hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
- }
-
- public Clause getClause() {
- return Clause.this;
- }
-
- @Override
- public int hashCode() {
- return hash;
- }
- //if the delta is lower , this violation is less serious
- public boolean isLessSerious(Violation that) {
- return that.delta != null && delta != null &&
- Math.abs(delta) < Math.abs(that.delta);
- }
-
- @Override
- public boolean equals(Object that) {
- if (that instanceof Violation) {
- Violation v = (Violation) that;
- return Objects.equals(this.shard, v.shard) &&
- Objects.equals(this.coll, v.coll) &&
- Objects.equals(this.node, v.node) &&
- Objects.equals(this.tagKey, v.tagKey)
- ;
- }
- return false;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.putIfNotNull("collection", coll);
- ew.putIfNotNull("shard", shard);
- ew.putIfNotNull("node", node);
- ew.putIfNotNull("tagKey", String.valueOf(tagKey));
- ew.putIfNotNull("violation", (MapWriter) ew1 -> {
- ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
- String.valueOf(actualVal));
- ew1.putIfNotNull("delta", delta);
- });
- ew.put("clause", getClause());
- }
- }
-
-
- public List<Violation> test(List<Row> allRows) {
- List<Violation> violations = new ArrayList<>();
- if (isPerCollectiontag()) {
- Map<String, Map<String, Map<String, AtomicInteger>>> replicaCount = computeReplicaCounts(allRows);
- for (Map.Entry<String, Map<String, Map<String, AtomicInteger>>> e : replicaCount.entrySet()) {
- if (!collection.isPass(e.getKey())) continue;
- for (Map.Entry<String, Map<String, AtomicInteger>> shardVsCount : e.getValue().entrySet()) {
- if (!shard.isPass(shardVsCount.getKey())) continue;
- for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
- if (!replica.isPass(counts.getValue())) {
- violations.add(new Violation(
- e.getKey(),
- shardVsCount.getKey(),
- tag.name.equals("node") ? counts.getKey() : null,
- counts.getValue(),
- replica.delta(counts.getValue()),
- counts.getKey()
- ));
- }
- }
- }
- }
- } else {
- for (Row r : allRows) {
- if (!tag.isPass(r)) {
- violations.add(new Violation(null, null, r.node, r.getVal(tag.name), tag.delta(r.getVal(tag.name)), null));
- }
- }
- }
- return violations;
-
- }
-
-
- private Map<String, Map<String, Map<String, AtomicInteger>>> computeReplicaCounts(List<Row> allRows) {
- Map<String, Map<String, Map<String, AtomicInteger>>> collVsShardVsTagVsCount = new HashMap<>();
- for (Row row : allRows)
- for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
- String collectionName = colls.getKey();
- if (!collection.isPass(collectionName)) continue;
- collVsShardVsTagVsCount.putIfAbsent(collectionName, new HashMap<>());
- Map<String, Map<String, AtomicInteger>> collMap = collVsShardVsTagVsCount.get(collectionName);
- for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
- String shardName = shards.getKey();
- if (ANY.equals(shard.val)) shardName = ANY;
- if (!shard.isPass(shardName)) break;
- collMap.putIfAbsent(shardName, new HashMap<>());
- Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
- Object tagVal = row.getVal(tag.name);
- tagVsCount.putIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", new AtomicInteger());
- if (tag.isPass(tagVal)) {
- tagVsCount.get(String.valueOf(tagVal)).addAndGet(shards.getValue().size());
- }
- }
- }
- return collVsShardVsTagVsCount;
- }
-
- public boolean isStrict() {
- return strict;
- }
-
- @Override
- public String toString() {
- return Utils.toJSONString(original);
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- for (Map.Entry<String, Object> e : original.entrySet()) ew.put(e.getKey(), e.getValue());
- }
-
- enum TestStatus {
- NOT_APPLICABLE, FAIL, PASS
- }
-
- private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict"));
-
- static class ValidateInfo {
- final Class type;
- final Set<String> vals;
- final Number min;
- final Number max;
-
-
- ValidateInfo(Class type, Set<String> vals, Number min, Number max) {
- this.type = type;
- this.vals = vals;
- this.min = min;
- if(min != null && !type.isInstance(min)) throw new RuntimeException("wrong min value type, expected: " + type.getName() + " actual: " + min.getClass().getName());
- this.max = max;
- if(max != null && !type.isInstance(max)) throw new RuntimeException("wrong max value type, expected: " + type.getName() + " actual: " + max.getClass().getName());
- }
- }
-
-
- /**
- *
- * @param name name of the condition
- * @param val value of the condition
- * @param isRuleVal is this provided in the rule
- * @return actual validated value
- */
- public static Object validate(String name, Object val, boolean isRuleVal) {
- if (val == null) return null;
- ValidateInfo info = validatetypes.get(name);
- if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = validatetypes.get("STRING");
- if (info == null) throw new RuntimeException("Unknown type :" + name);
- if (info.type == Double.class) {
- Double num = parseDouble(name, val);
- if (isRuleVal) {
- if (info.min != null)
- if (Double.compare(num, (Double) info.min) == -1)
- throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
- if (info.max != null)
- if (Double.compare(num, (Double) info.max) == 1)
- throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
- }
- return num;
- } else if (info.type == Long.class) {
- Long num = parseLong(name, val);
- if (isRuleVal) {
- if (info.min != null)
- if (num < info.min.longValue())
- throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
- if (info.max != null)
- if (num > info.max.longValue())
- throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
- }
- return num;
- } else if (info.type == String.class) {
- if (isRuleVal && info.vals != null && !info.vals.contains(val))
- throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(info.vals, ','));
- return val;
- } else {
- throw new RuntimeException("Invalid type ");
- }
- }
-
- public static Long parseLong(String name, Object val) {
- if (val == null) return null;
- if (val instanceof Long) return (Long) val;
- Number num = null;
- if (val instanceof String) {
- try {
- num = Long.parseLong(((String) val).trim());
- } catch (NumberFormatException e) {
- try {
- num = Double.parseDouble((String) val);
- } catch (NumberFormatException e1) {
- throw new RuntimeException(name + ": " + val + "not a valid number", e);
- }
- }
-
- } else if (val instanceof Number) {
- num = (Number) val;
- }
-
- if (num != null) {
- return num.longValue();
- }
- throw new RuntimeException(name + ": " + val + "not a valid number");
- }
-
- public static Double parseDouble(String name, Object val) {
- if (val == null) return null;
- if (val instanceof Double) return (Double) val;
- Number num = null;
- if (val instanceof String) {
- try {
- num = Double.parseDouble((String) val);
- } catch (NumberFormatException e) {
- throw new RuntimeException(name + ": " + val + "not a valid number", e);
- }
-
- } else if (val instanceof Number) {
- num = (Number) val;
- }
-
- if (num != null) {
- return num.doubleValue();
- }
- throw new RuntimeException(name + ": " + val + "not a valid number");
- }
-
- private static final Map<String, ValidateInfo> validatetypes = new HashMap<>();
-
- static {
- validatetypes.put("collection", new ValidateInfo(String.class, null, null, null));
- validatetypes.put("shard", new ValidateInfo(String.class, null, null, null));
- validatetypes.put("replica", new ValidateInfo(Long.class, null, 0L, null));
- validatetypes.put(ImplicitSnitch.PORT, new ValidateInfo(Long.class, null, 1L, 65535L));
- validatetypes.put(ImplicitSnitch.DISK, new ValidateInfo(Double.class, null, 0d, Double.MAX_VALUE));
- validatetypes.put(ImplicitSnitch.NODEROLE, new ValidateInfo(String.class, Collections.singleton("overseer"), null, null));
- validatetypes.put(ImplicitSnitch.CORES, new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));
- validatetypes.put(ImplicitSnitch.SYSLOADAVG, new ValidateInfo(Double.class, null, 0d, 100d));
- validatetypes.put(ImplicitSnitch.HEAPUSAGE, new ValidateInfo(Double.class, null, 0d, null));
- validatetypes.put("NUMBER", new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));//generic number validation
- validatetypes.put("STRING", new ValidateInfo(String.class, null, null, null));//generic string validation
- validatetypes.put("node", new ValidateInfo(String.class, null, null, null));
- for (String ip : ImplicitSnitch.IP_SNITCHES) validatetypes.put(ip, new ValidateInfo(Long.class, null, 0L, 255L));
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java
deleted file mode 100644
index 710db87..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-public interface ClusterDataProvider extends Closeable {
- /**Get the value of each tag for a given node
- *
- * @param node node name
- * @param tags tag names
- * @return a map of tag vs value
- */
- Map<String, Object> getNodeValues(String node, Collection<String> tags);
-
- /**
- * Get the details of each replica in a node. It attempts to fetch as much details about
- * the replica as mentioned in the keys list. It is not necessary to give al details
- * <p>
- * the format is {collection:shard :[{replicadetails}]}
- */
- Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
-
- Collection<String> getNodes();
-
- /**Get the collection-specific policy
- */
- String getPolicyNameByCollection(String coll);
-
- @Override
- default void close() throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
deleted file mode 100644
index 97aef51..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.util.List;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.autoscaling.Clause.Violation;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
-import org.apache.solr.cloud.autoscaling.Policy.Suggester;
-import org.apache.solr.common.util.Pair;
-
-public class MoveReplicaSuggester extends Suggester {
-
- @Override
- SolrRequest init() {
- SolrRequest operation = tryEachNode(true);
- if (operation == null) operation = tryEachNode(false);
- return operation;
- }
-
- SolrRequest tryEachNode(boolean strict) {
- //iterate through elements and identify the least loaded
- List<Clause.Violation> leastSeriousViolation = null;
- Integer targetNodeIndex = null;
- Integer fromNodeIndex = null;
- ReplicaInfo fromReplicaInfo = null;
- for (Pair<ReplicaInfo, Row> fromReplica : getValidReplicas(true, true, -1)) {
- Row fromRow = fromReplica.second();
- ReplicaInfo replicaInfo = fromReplica.first();
- String coll = replicaInfo.collection;
- String shard = replicaInfo.shard;
- Pair<Row, ReplicaInfo> pair = fromRow.removeReplica(coll, shard);
- Row tmpRow = pair.first();
- if (tmpRow == null) {
- //no such replica available
- continue;
- }
- tmpRow.violations.clear();
-
- final int i = getMatrix().indexOf(fromRow);
- for (int j = getMatrix().size() - 1; j > i; j--) {
- Row targetRow = getMatrix().get(j);
- if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
- targetRow = targetRow.addReplica(coll, shard);
- targetRow.violations.clear();
- List<Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getModifiedMatrix(getMatrix(), tmpRow, i), targetRow, j));
- if (!containsNewErrors(errs) && isLessSerious(errs, leastSeriousViolation)) {
- leastSeriousViolation = errs;
- targetNodeIndex = j;
- fromNodeIndex = i;
- fromReplicaInfo = replicaInfo;
- }
- }
- }
- if (targetNodeIndex != null && fromNodeIndex != null) {
- getMatrix().set(fromNodeIndex, getMatrix().get(fromNodeIndex).removeReplica(fromReplicaInfo.collection, fromReplicaInfo.shard).first());
- getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(fromReplicaInfo.collection, fromReplicaInfo.shard));
- return new CollectionAdminRequest.MoveReplica(
- fromReplicaInfo.collection,
- fromReplicaInfo.name,
- getMatrix().get(targetNodeIndex).node);
- }
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
deleted file mode 100644
index f961cac..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.util.Objects;
-
-import org.apache.solr.cloud.autoscaling.Clause.TestStatus;
-
-import static org.apache.solr.cloud.autoscaling.Clause.TestStatus.FAIL;
-import static org.apache.solr.cloud.autoscaling.Clause.TestStatus.NOT_APPLICABLE;
-import static org.apache.solr.cloud.autoscaling.Clause.TestStatus.PASS;
-import static org.apache.solr.cloud.autoscaling.Policy.ANY;
-
-
-public enum Operand {
- WILDCARD(ANY, Integer.MAX_VALUE) {
- @Override
- public TestStatus match(Object ruleVal, Object testVal) {
- return testVal == null ? NOT_APPLICABLE : PASS;
- }
-
- },
- EQUAL("", 0) {
- @Override
- public int _delta(int expected, int actual) {
- return expected - actual;
- }
- },
- NOT_EQUAL("!", 2) {
- @Override
- public TestStatus match(Object ruleVal, Object testVal) {
- return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
- }
-
- @Override
- public int _delta(int expected, int actual) {
- return expected - actual;
- }
-
- },
- GREATER_THAN(">", 1) {
- @Override
- public TestStatus match(Object ruleVal, Object testVal) {
- if (testVal == null) return NOT_APPLICABLE;
- if (ruleVal instanceof Double) {
- return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == 1 ? PASS : FAIL;
- }
- return getLong(testVal) > getLong(ruleVal) ? PASS: FAIL ;
- }
-
- @Override
- protected int _delta(int expected, int actual) {
- return actual > expected ? 0 : (expected + 1) - actual;
- }
- },
- LESS_THAN("<", 2) {
- @Override
- public TestStatus match(Object ruleVal, Object testVal) {
- if (testVal == null) return NOT_APPLICABLE;
- if (ruleVal instanceof Double) {
- return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == -1 ? PASS : FAIL;
- }
- return getLong(testVal) < getLong(ruleVal) ? PASS: FAIL ;
- }
-
- @Override
- protected int _delta(int expected, int actual) {
- return actual < expected ? 0 : (expected ) - actual;
- }
-
- };
- public final String operand;
- final int priority;
-
- Operand(String val, int priority) {
- this.operand = val;
- this.priority = priority;
- }
-
- public String toStr(Object expectedVal) {
- return operand + expectedVal.toString();
- }
-
- public TestStatus match(Object ruleVal, Object testVal) {
- return Objects.equals(ruleVal, testVal) ? PASS : FAIL;
- }
-
- Long getLong(Object o) {
- if (o instanceof Long) return (Long) o;
- if(o instanceof Number ) return ((Number) o).longValue();
- return Long.parseLong(String.valueOf(o));
-
- }
-
- public Integer delta(Object expected, Object actual) {
- try {
- Integer expectedInt = Integer.parseInt(String.valueOf(expected));
- Integer actualInt = Integer.parseInt(String.valueOf(actual));
- return _delta(expectedInt, actualInt);
- } catch (Exception e) {
- return null;
- }
- }
-
- protected int _delta(int expected, int actual) {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
deleted file mode 100644
index c86ccf9..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ /dev/null
@@ -1,519 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.cloud.autoscaling.Clause.Violation;
-import org.apache.solr.common.IteratorWriter;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.params.CollectionParams.CollectionAction;
-import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.Utils;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static java.util.stream.Collectors.toList;
-
-/*The class that reads, parses and applies policies specified in
- * autoscaling.json
- *
- * Create one instance of this class per unique autoscaling.json.
- * This is immutable and is thread-safe
- *
- * Create a fresh new session for each use
- *
- */
-public class Policy implements MapWriter {
- public static final String POLICY = "policy";
- public static final String EACH = "#EACH";
- public static final String ANY = "#ANY";
- public static final String CLUSTER_POLICY = "cluster-policy";
- public static final String CLUSTER_PREFERENCE = "cluster-preferences";
- public static final Set<String> GLOBAL_ONLY_TAGS = Collections.singleton("cores");
- final Map<String, List<Clause>> policies = new HashMap<>();
- final List<Clause> clusterPolicy;
- final List<Preference> clusterPreferences;
- final List<String> params;
-
-
- public Policy(Map<String, Object> jsonMap) {
-
- clusterPreferences = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_PREFERENCE, emptyList())).stream()
- .map(Preference::new)
- .collect(toList());
- for (int i = 0; i < clusterPreferences.size() - 1; i++) {
- Preference preference = clusterPreferences.get(i);
- preference.next = clusterPreferences.get(i + 1);
- }
- if (clusterPreferences.isEmpty()) {
- clusterPreferences.add(new Preference((Map<String, Object>) Utils.fromJSONString("{minimize : cores, precision:1}")));
- }
- SortedSet<String> paramsOfInterest = new TreeSet<>();
- for (Preference preference : clusterPreferences) {
- if (paramsOfInterest.contains(preference.name.name())) {
- throw new RuntimeException(preference.name + " is repeated");
- }
- paramsOfInterest.add(preference.name.toString());
- }
- this.params = new ArrayList<>(paramsOfInterest);
-
- clusterPolicy = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_POLICY, emptyList())).stream()
- .map(Clause::new)
- .filter(clause -> {
- clause.addTags(params);
- return true;
- })
- .collect(Collectors.toList());
-
- ((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault("policies", emptyMap())).forEach((s, l1) ->
- this.policies.put(s, l1.stream()
- .map(Clause::new)
- .filter(clause -> {
- if (!clause.isPerCollectiontag())
- throw new RuntimeException(clause.globalTag.name + " is only allowed in 'cluster-policy'");
- clause.addTags(params);
- return true;
- })
- .sorted()
- .collect(toList())));
- }
-
- public List<Clause> getClusterPolicy() {
- return clusterPolicy;
- }
-
- public List<Preference> getClusterPreferences() {
- return clusterPreferences;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- if (!policies.isEmpty()) {
- ew.put("policies", (MapWriter) ew1 -> {
- for (Map.Entry<String, List<Clause>> e : policies.entrySet()) {
- ew1.put(e.getKey(), e.getValue());
- }
- });
- }
- if (!clusterPreferences.isEmpty()) {
- ew.put("preferences", (IteratorWriter) iw -> {
- for (Preference p : clusterPreferences) iw.add(p);
- });
- }
-
- }
-
- /*This stores the logical state of the system, given a policy and
- * a cluster state.
- *
- */
- public class Session implements MapWriter {
- final List<String> nodes;
- final ClusterDataProvider dataProvider;
- final List<Row> matrix;
- Set<String> collections = new HashSet<>();
- List<Clause> expandedClauses;
- List<Violation> violations = new ArrayList<>();
-
- private Session(List<String> nodes, ClusterDataProvider dataProvider,
- List<Row> matrix, List<Clause> expandedClauses) {
- this.nodes = nodes;
- this.dataProvider = dataProvider;
- this.matrix = matrix;
- this.expandedClauses = expandedClauses;
- }
-
- Session(ClusterDataProvider dataProvider) {
- this.nodes = new ArrayList<>(dataProvider.getNodes());
- this.dataProvider = dataProvider;
- for (String node : nodes) {
- collections.addAll(dataProvider.getReplicaInfo(node, Collections.emptyList()).keySet());
- }
-
- expandedClauses = clusterPolicy.stream()
- .filter(clause -> !clause.isPerCollectiontag())
- .collect(Collectors.toList());
-
- for (String c : collections) {
- addClausesForCollection(dataProvider, c);
- }
-
- Collections.sort(expandedClauses);
-
- matrix = new ArrayList<>(nodes.size());
- for (String node : nodes) matrix.add(new Row(node, params, dataProvider));
- applyRules();
- }
-
- private void addClausesForCollection(ClusterDataProvider dataProvider, String c) {
- String p = dataProvider.getPolicyNameByCollection(c);
- if (p != null) {
- List<Clause> perCollPolicy = policies.get(p);
- if (perCollPolicy == null)
- throw new RuntimeException(StrUtils.formatString("Policy for collection {0} is {1} . It does not exist", c, p));
- }
- expandedClauses.addAll(mergePolicies(c, policies.getOrDefault(p, emptyList()), clusterPolicy));
- }
-
- Session copy() {
- return new Session(nodes, dataProvider, getMatrixCopy(), expandedClauses);
- }
-
- List<Row> getMatrixCopy() {
- return matrix.stream()
- .map(Row::copy)
- .collect(Collectors.toList());
- }
-
- Policy getPolicy() {
- return Policy.this;
-
- }
-
- /**
- * Apply the preferences and conditions
- */
- private void applyRules() {
- if (!clusterPreferences.isEmpty()) {
- //this is to set the approximate value according to the precision
- ArrayList<Row> tmpMatrix = new ArrayList<>(matrix);
- for (Preference p : clusterPreferences) {
- Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false));
- p.setApproxVal(tmpMatrix);
- }
- //approximate values are set now. Let's do recursive sorting
- Collections.sort(matrix, (Row r1, Row r2) -> {
- int result = clusterPreferences.get(0).compare(r1, r2, true);
- if (result == 0) result = clusterPreferences.get(0).compare(r1, r2, false);
- return result;
- });
- }
-
- for (Clause clause : expandedClauses) {
- List<Violation> errs = clause.test(matrix);
- violations.addAll(errs);
- }
- }
-
- public List<Violation> getViolations() {
- return violations;
- }
-
- public Suggester getSuggester(CollectionAction action) {
- Suggester op = ops.get(action).get();
- if (op == null) throw new UnsupportedOperationException(action.toString() + "is not supported");
- op._init(this);
- return op;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- for (int i = 0; i < matrix.size(); i++) {
- Row row = matrix.get(i);
- ew.put(row.node, row);
- }
- }
-
- @Override
- public String toString() {
- return Utils.toJSONString(toMap(new LinkedHashMap<>()));
- }
-
- public List<Row> getSorted() {
- return Collections.unmodifiableList(matrix);
- }
- }
-
-
- public Session createSession(ClusterDataProvider dataProvider) {
- return new Session(dataProvider);
- }
-
- enum SortParam {
- freedisk(0, Integer.MAX_VALUE), cores(0, Integer.MAX_VALUE), heapUsage(0, Integer.MAX_VALUE), sysLoadAvg(0, 100);
-
- public final int min,max;
-
- SortParam(int min, int max) {
- this.min = min;
- this.max = max;
- }
-
- static SortParam get(String m) {
- for (SortParam p : values()) if (p.name().equals(m)) return p;
- throw new RuntimeException(StrUtils.formatString("Invalid sort {0} Sort must be on one of these {1}", m, Arrays.asList(values())));
- }
- }
-
- enum Sort {
- maximize(1), minimize(-1);
- final int sortval;
-
- Sort(int i) {
- sortval = i;
- }
-
- static Sort get(Map<String, Object> m) {
- if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) {
- throw new RuntimeException("Cannot have both 'maximize' and 'minimize'");
- }
- if (m.containsKey(maximize.name())) return maximize;
- if (m.containsKey(minimize.name())) return minimize;
- throw new RuntimeException("must have either 'maximize' or 'minimize'");
- }
- }
-
-
- public static class ReplicaInfo implements MapWriter {
- final String name;
- String core, collection, shard;
- Map<String, Object> variables;
-
- public ReplicaInfo(String name, String coll, String shard, Map<String, Object> vals) {
- this.name = name;
- this.variables = vals;
- this.collection = coll;
- this.shard = shard;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(name, variables);
- }
-
- public String getCore() {
- return core;
- }
-
- public String getCollection() {
- return collection;
- }
-
- public String getShard() {
- return shard;
- }
- }
-
-
- /* A suggester is capable of suggesting a collection operation
- * given a particular session. Before it suggests a new operation,
- * it ensures that ,
- * a) load is reduced on the most loaded node
- * b) it causes no new violations
- *
- */
- public static abstract class Suggester {
- protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
- Policy.Session session;
- SolrRequest operation;
- protected List<Violation> originalViolations = new ArrayList<>();
- private boolean isInitialized = false;
-
- private void _init(Session session) {
- this.session = session.copy();
- }
-
- public Suggester hint(Hint hint, Object value) {
- if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
- ((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).add(value);
- } else {
- hints.put(hint, value);
- }
- return this;
- }
-
- abstract SolrRequest init();
-
-
- public SolrRequest getOperation() {
- if (!isInitialized) {
- String coll = (String) hints.get(Hint.COLL);
- String shard = (String) hints.get(Hint.SHARD);
- // if this is not a known collection from the existing clusterstate,
- // then add it
- if (session.matrix.stream().noneMatch(row -> row.collectionVsShardVsReplicas.containsKey(coll))) {
- session.addClausesForCollection(session.dataProvider, coll);
- Collections.sort(session.expandedClauses);
- }
- if (coll != null) {
- for (Row row : session.matrix) {
- if (!row.collectionVsShardVsReplicas.containsKey(coll)) row.collectionVsShardVsReplicas.put(coll, new HashMap<>());
- if (shard != null) {
- Map<String, List<ReplicaInfo>> shardInfo = row.collectionVsShardVsReplicas.get(coll);
- if (!shardInfo.containsKey(shard)) shardInfo.put(shard, new ArrayList<>());
- }
- }
- }
- session.applyRules();
- originalViolations.addAll(session.getViolations());
- this.operation = init();
- isInitialized = true;
- }
- return operation;
- }
-
- public Session getSession() {
- return session;
- }
-
- List<Row> getMatrix() {
- return session.matrix;
-
- }
-
- //check if the fresh set of violations is less serious than the last set of violations
- boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
- if (old == null || fresh.size() < old.size()) return true;
- if (fresh.size() == old.size()) {
- for (int i = 0; i < fresh.size(); i++) {
- Violation freshViolation = fresh.get(i);
- Violation oldViolation = null;
- for (Violation v : old) {
- if (v.equals(freshViolation)) oldViolation = v;
- }
- if (oldViolation != null && freshViolation.isLessSerious(oldViolation)) return true;
- }
- }
- return false;
- }
-
- boolean containsNewErrors(List<Violation> violations) {
- for (Violation v : violations) {
- int idx = originalViolations.indexOf(v);
- if (idx < 0 || originalViolations.get(idx).isLessSerious(v)) return true;
- }
- return false;
- }
-
- List<Pair<ReplicaInfo, Row>> getValidReplicas(boolean sortDesc, boolean isSource, int until) {
- List<Pair<Policy.ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
-
- if (sortDesc) {
- if (until == -1) until = getMatrix().size();
- for (int i = 0; i < until; i++) addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
- } else {
- if (until == -1) until = 0;
- for (int i = getMatrix().size() - 1; i >= until; i--)
- addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
- }
- return allPossibleReplicas;
- }
-
- void addReplicaToList(Row r, boolean isSource, List<Pair<Policy.ReplicaInfo, Row>> replicaList) {
- if (!isAllowed(r.node, isSource ? Hint.SRC_NODE : Hint.TARGET_NODE)) return;
- for (Map.Entry<String, Map<String, List<Policy.ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
- if (!isAllowed(e.getKey(), Hint.COLL)) continue;
- for (Map.Entry<String, List<Policy.ReplicaInfo>> shard : e.getValue().entrySet()) {
- if (!isAllowed(e.getKey(), Hint.SHARD)) continue;
- replicaList.add(new Pair<>(shard.getValue().get(0), r));
- }
- }
- }
-
- protected List<Violation> testChangedMatrix(boolean strict, List<Row> rows) {
- List<Violation> errors = new ArrayList<>();
- for (Clause clause : session.expandedClauses) {
- if (strict || clause.strict) {
- List<Violation> errs = clause.test(rows);
- if (!errs.isEmpty()) {
- errors.addAll(errs);
- }
- }
- }
- return errors;
- }
-
- ArrayList<Row> getModifiedMatrix(List<Row> matrix, Row tmpRow, int i) {
- ArrayList<Row> copy = new ArrayList<>(matrix);
- copy.set(i, tmpRow);
- return copy;
- }
-
- protected boolean isAllowed(Object v, Hint hint) {
- Object hintVal = hints.get(hint);
- if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
- Set set = (Set) hintVal;
- return set == null || set.contains(v);
- } else {
- return hintVal == null || Objects.equals(v, hintVal);
- }
- }
-
- public enum Hint {
- COLL, SHARD, SRC_NODE, TARGET_NODE
- }
-
-
- }
-
- static List<Clause> mergePolicies(String coll,
- List<Clause> collPolicy,
- List<Clause> globalPolicy) {
-
- List<Clause> merged = insertColl(coll, collPolicy);
- List<Clause> global = insertColl(coll, globalPolicy);
- merged.addAll(global.stream()
- .filter(clusterPolicyClause -> merged.stream().noneMatch(perCollPolicy -> perCollPolicy.doesOverride(clusterPolicyClause)))
- .collect(Collectors.toList()));
- return merged;
- }
-
- /**
- * Insert the collection name into the clauses where collection is not specified
- */
- static List<Clause> insertColl(String coll, Collection<Clause> conditions) {
- return conditions.stream()
- .filter(Clause::isPerCollectiontag)
- .map(clause -> {
- Map<String, Object> copy = new LinkedHashMap<>(clause.original);
- if (!copy.containsKey("collection")) copy.put("collection", coll);
- return new Clause(copy);
- })
- .filter(it -> (it.collection.isPass(coll)))
- .collect(Collectors.toList());
-
- }
-
- private static final Map<CollectionAction, Supplier<Suggester>> ops = new HashMap<>();
-
- static {
- ops.put(CollectionAction.ADDREPLICA, () -> new AddReplicaSuggester());
- ops.put(CollectionAction.MOVEREPLICA, () -> new MoveReplicaSuggester());
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
deleted file mode 100644
index 33d4b97..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.cloud.autoscaling.Policy.Suggester.Hint;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.CoreAdminParams;
-import org.apache.solr.common.util.Utils;
-
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
-
-public class PolicyHelper {
- public static Map<String, List<String>> getReplicaLocations(String collName, Map<String, Object> autoScalingJson,
- ClusterDataProvider cdp,
- Map<String, String> optionalPolicyMapping,
- List<String> shardNames,
- int repFactor,
- List<String> nodesList) {
- Map<String, List<String>> positionMapping = new HashMap<>();
- for (String shardName : shardNames) positionMapping.put(shardName, new ArrayList<>(repFactor));
- if (optionalPolicyMapping != null) {
- final ClusterDataProvider delegate = cdp;
- cdp = new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return delegate.getNodeValues(node, tags);
- }
-
- @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return delegate.getReplicaInfo(node, keys);
- }
-
- @Override
- public Collection<String> getNodes() {
- return delegate.getNodes();
- }
-
- @Override
- public String getPolicyNameByCollection(String coll) {
- return optionalPolicyMapping.containsKey(coll) ?
- optionalPolicyMapping.get(coll) :
- delegate.getPolicyNameByCollection(coll);
- }
- };
-
- }
-
-
- Policy policy = new Policy(autoScalingJson);
- Policy.Session session = policy.createSession(cdp);
- for (String shardName : shardNames) {
- for (int i = 0; i < repFactor; i++) {
- Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
- .hint(Hint.COLL, collName)
- .hint(Hint.SHARD, shardName);
- if (nodesList != null) {
- for (String nodeName : nodesList) {
- suggester = suggester.hint(Hint.TARGET_NODE, nodeName);
- }
- }
- SolrRequest op = suggester.getOperation();
- if (op == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(Utils.getDeepCopy(session.expandedClauses, 4, true)));
- }
- session = suggester.getSession();
- positionMapping.get(shardName).add(op.getParams().get(CoreAdminParams.NODE));
- }
- }
-
- return positionMapping;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Preference.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Preference.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Preference.java
deleted file mode 100644
index 0566d25..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Preference.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.Utils;
-
-class Preference implements MapWriter {
- final Policy.SortParam name;
- Integer precision;
- final Policy.Sort sort;
- Preference next;
- public int idx;
- private final Map original;
-
- Preference(Map<String, Object> m) {
- this.original = Utils.getDeepCopy(m,3);
- sort = Policy.Sort.get(m);
- name = Policy.SortParam.get(m.get(sort.name()).toString());
- Object p = m.getOrDefault("precision", 0);
- precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
- if (precision < 0) {
- throw new RuntimeException("precision must be a positive value ");
- }
- if(precision< name.min || precision> name.max){
- throw new RuntimeException(StrUtils.formatString("invalid precision value {0} must lie between {1} and {1}",
- precision, name.min, name.max ) );
- }
-
- }
-
- // there are 2 modes of compare.
- // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
- // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
- int compare(Row r1, Row r2, boolean useApprox) {
- Object o1 = useApprox ? r1.cells[idx].approxVal : r1.cells[idx].val;
- Object o2 = useApprox ? r2.cells[idx].approxVal : r2.cells[idx].val;
- int result = 0;
- if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
- else if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
- else if (!o1.getClass().getName().equals(o2.getClass().getName())) {
- throw new RuntimeException("Unable to compare " + o1 + " of type: " + o1.getClass().getName() + " from " + r1.cells[idx].toString() + " and " + o2 + " of type: " + o2.getClass().getName() + " from " + r2.cells[idx].toString());
- }
- return result == 0 ? (next == null ? 0 : next.compare(r1, r2, useApprox)) : sort.sortval * result;
- }
-
- //sets the new value according to precision in val_
- void setApproxVal(List<Row> tmpMatrix) {
- Object prevVal = null;
- for (Row row : tmpMatrix) {
- prevVal = row.cells[idx].approxVal =
- (prevVal == null || Double.compare(Math.abs(((Number) prevVal).doubleValue() - ((Number) row.cells[idx].val).doubleValue()), precision) > 0) ?
- row.cells[idx].val :
- prevVal;
- }
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- for (Object o : original.entrySet()) {
- Map.Entry e = (Map.Entry) o;
- ew.put(String.valueOf(e.getKey()), e.getValue());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
deleted file mode 100644
index f7ab5ca..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.solr.common.IteratorWriter;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
-
-import static org.apache.solr.common.params.CoreAdminParams.NODE;
-
-
-class Row implements MapWriter {
- public final String node;
- final Cell[] cells;
- Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas;
- List<Clause> violations = new ArrayList<>();
- boolean anyValueMissing = false;
-
- Row(String node, List<String> params, ClusterDataProvider dataProvider) {
- collectionVsShardVsReplicas = dataProvider.getReplicaInfo(node, params);
- if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>();
- this.node = node;
- cells = new Cell[params.size()];
- Map<String, Object> vals = dataProvider.getNodeValues(node, params);
- for (int i = 0; i < params.size(); i++) {
- String s = params.get(i);
- cells[i] = new Cell(i, s, Clause.validate(s,vals.get(s), false));
- if (NODE.equals(s)) cells[i].val = node;
- if (cells[i].val == null) anyValueMissing = true;
- }
- }
-
- Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, List<Clause> violations) {
- this.node = node;
- this.cells = new Cell[cells.length];
- for (int i = 0; i < this.cells.length; i++) {
- this.cells[i] = cells[i].copy();
-
- }
- this.anyValueMissing = anyValueMissing;
- this.collectionVsShardVsReplicas = collectionVsShardVsReplicas;
- this.violations = violations;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(node, (IteratorWriter) iw -> {
- iw.add((MapWriter) e -> e.put("replicas", collectionVsShardVsReplicas));
- for (Cell cell : cells) iw.add(cell);
- });
- }
-
- Row copy() {
- return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(collectionVsShardVsReplicas, 3), new ArrayList<>(violations));
- }
-
- Object getVal(String name) {
- for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
- return null;
- }
-
- @Override
- public String toString() {
- return node;
- }
-
- // this adds a replica to the replica info
- Row addReplica(String coll, String shard) {
- Row row = copy();
- Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
- List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
- replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, new HashMap<>()));
- for (Cell cell : row.cells) {
- if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() + 1;
- }
- return row;
-
- }
-
- Pair<Row, ReplicaInfo> removeReplica(String coll, String shard) {
- Row row = copy();
- Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.get(coll);
- if (c == null) return null;
- List<ReplicaInfo> s = c.get(shard);
- if (s == null || s.isEmpty()) return null;
- for (Cell cell : row.cells) {
- if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() -1;
- }
- return new Pair(row, s.remove(0));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/package-info.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/package-info.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/package-info.java
deleted file mode 100644
index 472b1ac..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Common classes for autoscaling parsing filtering nodes and sorting
- */
-
-package org.apache.solr.cloud.autoscaling;
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
index 5dc4ebb..b1d6ee2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
@@ -28,7 +28,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.function.BiConsumer;
-import org.apache.solr.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.noggit.JSONUtil;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 8c296b9..a036c8d 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -31,9 +31,16 @@ import java.util.Map;
import com.google.common.collect.ImmutableList;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Cell;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
+import org.apache.solr.client.solrj.cloud.autoscaling.Operand;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
+import org.apache.solr.client.solrj.cloud.autoscaling.Row;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.autoscaling.Clause.Violation;
-import org.apache.solr.cloud.autoscaling.Policy.Suggester.Hint;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.Utils;
@@ -219,7 +226,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
" }" +
"}");
Policy policy = new Policy(map);
- List<Clause> clauses = Policy.mergePolicies("mycoll", policy.policies.get("policy1"), policy.clusterPolicy);
+ List<Clause> clauses = Policy.mergePolicies("mycoll", policy.getPolicies().get("policy1"), policy.getClusterPolicy());
Collections.sort(clauses);
assertEquals(clauses.size(), 4);
assertEquals("1", String.valueOf(clauses.get(0).original.get("replica")));
@@ -240,8 +247,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
Policy p = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
List<Clause> clauses = new ArrayList<>(p.getClusterPolicy());
Collections.sort(clauses);
- assertEquals("nodeRole", clauses.get(1).tag.name);
- assertEquals("sysprop.rack", clauses.get(0).tag.name);
+ assertEquals("nodeRole", clauses.get(1).tag.getName());
+ assertEquals("sysprop.rack", clauses.get(0).tag.getName());
}
public void testRules() throws IOException {
@@ -275,9 +282,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<Violation> violations = session.getViolations();
assertEquals(3, violations.size());
- assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.val)));
- assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.name)));
- assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.op == Operand.LESS_THAN && "node".equals(violation.getClause().tag.name))));
+ assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.getValue())));
+ assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.getName())));
+ assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.getOperand() == Operand.LESS_THAN && "node".equals(violation.getClause().tag.getName()))));
Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
.hint(Hint.COLL, "gettingstarted")
[16/18] lucene-solr:feature/autoscaling: SOLR-10931: Move TestPolicy
to org.apache.solr.client.solrj.cloud.autoscaling package
Posted by sh...@apache.org.
SOLR-10931: Move TestPolicy to org.apache.solr.client.solrj.cloud.autoscaling package
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b99ee2b1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b99ee2b1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b99ee2b1
Branch: refs/heads/feature/autoscaling
Commit: b99ee2b136703d18b3c445493fb72f54ccb6fd7f
Parents: 13a3ae2
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Jun 28 11:53:27 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Jun 28 11:53:27 2017 +0530
----------------------------------------------------------------------
.../solrj/cloud/autoscaling/TestPolicy.java | 654 ++++++++++++++++++
.../solr/cloud/autoscaling/TestPolicy.java | 661 -------------------
2 files changed, 654 insertions(+), 661 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b99ee2b1/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
new file mode 100644
index 0000000..d17e7ce
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.common.util.ValidatingJsonMap;
+
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+
+public class TestPolicy extends SolrTestCaseJ4 {
+
+ public static String clusterState = "{'gettingstarted':{" +
+ " 'router':{'name':'compositeId'}," +
+ " 'shards':{" +
+ " 'shard1':{" +
+ " 'range':'80000000-ffffffff'," +
+ " 'replicas':{" +
+ " 'r1':{" +
+ " 'core':r1," +
+ " 'base_url':'http://10.0.0.4:8983/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'," +
+ " 'leader':'true'}," +
+ " 'r2':{" +
+ " 'core':r2," +
+ " 'base_url':'http://10.0.0.4:7574/solr'," +
+ " 'node_name':'node2'," +
+ " 'state':'active'}}}," +
+ " 'shard2':{" +
+ " 'range':'0-7fffffff'," +
+ " 'replicas':{" +
+ " 'r3':{" +
+ " 'core':r3," +
+ " 'base_url':'http://10.0.0.4:8983/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'," +
+ " 'leader':'true'}," +
+ " 'r4':{" +
+ " 'core':r4," +
+ " 'base_url':'http://10.0.0.4:8987/solr'," +
+ " 'node_name':'node4'," +
+ " 'state':'active'}," +
+ " 'r6':{" +
+ " 'core':r6," +
+ " 'base_url':'http://10.0.0.4:8989/solr'," +
+ " 'node_name':'node3'," +
+ " 'state':'active'}," +
+ " 'r5':{" +
+ " 'core':r5," +
+ " 'base_url':'http://10.0.0.4:7574/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'}}}}}}";
+
+ public static Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaDetails(String node, String s) {
+ ValidatingJsonMap m = ValidatingJsonMap
+ .getDeepCopy((Map) Utils.fromJSONString(s), 6, true);
+ Map<String, Map<String, List<Policy.ReplicaInfo>>> result = new LinkedHashMap<>();
+
+ m.forEach((collName, o) -> {
+ ValidatingJsonMap coll = (ValidatingJsonMap) o;
+ coll.getMap("shards").forEach((shard, o1) -> {
+ ValidatingJsonMap sh = (ValidatingJsonMap) o1;
+ sh.getMap("replicas").forEach((replicaName, o2) -> {
+ ValidatingJsonMap r = (ValidatingJsonMap) o2;
+ String node_name = (String) r.get("node_name");
+ if (!node_name.equals(node)) return;
+ Map<String, List<Policy.ReplicaInfo>> shardVsReplicaStats = result.get(collName);
+ if (shardVsReplicaStats == null) result.put(collName, shardVsReplicaStats = new HashMap<>());
+ List<Policy.ReplicaInfo> replicaInfos = shardVsReplicaStats.get(shard);
+ if (replicaInfos == null) shardVsReplicaStats.put(shard, replicaInfos = new ArrayList<>());
+ replicaInfos.add(new Policy.ReplicaInfo(replicaName, collName, shard, new HashMap<>()));
+ });
+ });
+ });
+ return result;
+ }
+
+ public void testValidate() {
+ expectError("replica", -1, "must be greater than" );
+ expectError("replica","hello", "not a valid number" );
+ assertEquals( 1l, Clause.validate("replica", "1", true));
+ assertEquals("c", Clause.validate("collection", "c", true));
+ assertEquals( "s", Clause.validate("shard", "s",true));
+ assertEquals( "overseer", Clause.validate("nodeRole", "overseer",true));
+
+ expectError("nodeRole", "wrong","must be one of");
+
+ expectError("sysLoadAvg", "101","must be less than ");
+ expectError("sysLoadAvg", 101,"must be less than ");
+ expectError("sysLoadAvg", "-1","must be greater than");
+ expectError("sysLoadAvg", -1,"must be greater than");
+
+ assertEquals(12.46d,Clause.validate("sysLoadAvg", "12.46",true));
+ assertEquals(12.46,Clause.validate("sysLoadAvg", 12.46d,true));
+
+
+ expectError("ip_1", "300","must be less than ");
+ expectError("ip_1", 300,"must be less than ");
+ expectError("ip_1", "-1","must be greater than");
+ expectError("ip_1", -1,"must be greater than");
+
+ assertEquals(1l,Clause.validate("ip_1", "1",true));
+
+ expectError("heapUsage", "-1","must be greater than");
+ expectError("heapUsage", -1,"must be greater than");
+ assertEquals(69.9d,Clause.validate("heapUsage", "69.9",true));
+ assertEquals(69.9d,Clause.validate("heapUsage", 69.9d,true));
+
+ expectError("port", "70000","must be less than ");
+ expectError("port", 70000,"must be less than ");
+ expectError("port", "0","must be greater than");
+ expectError("port", 0,"must be greater than");
+
+ expectError("cores", "-1","must be greater than");
+
+
+ }
+
+ private static void expectError(String name, Object val, String msg){
+ try {
+ Clause.validate(name, val,true);
+ fail("expected exception containing "+msg);
+ } catch (Exception e) {
+ assertTrue("expected exception containing "+msg,e.getMessage().contains(msg));
+ }
+
+ }
+
+ public void testOperands() {
+ Clause c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:'<2', node:'#ANY'}"));
+ assertFalse(c.replica.isPass(3));
+ assertFalse(c.replica.isPass(2));
+ assertTrue(c.replica.isPass(1));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:'>2', node:'#ANY'}"));
+ assertTrue(c.replica.isPass(3));
+ assertFalse(c.replica.isPass(2));
+ assertFalse(c.replica.isPass(1));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}"));
+ assertTrue(c.tag.isPass("OVERSEER"));
+ assertFalse(c.tag.isPass("overseer"));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'<12.7'}"));
+ assertTrue(c.tag.isPass("12.6"));
+ assertTrue(c.tag.isPass(12.6d));
+ assertFalse(c.tag.isPass("12.9"));
+ assertFalse(c.tag.isPass(12.9d));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'>12.7'}"));
+ assertTrue(c.tag.isPass("12.8"));
+ assertTrue(c.tag.isPass(12.8d));
+ assertFalse(c.tag.isPass("12.6"));
+ assertFalse(c.tag.isPass(12.6d));
+ }
+
+ public void testRow() {
+ Row row = new Row("nodex", new Cell[]{new Cell(0, "node", "nodex")}, false, new HashMap<>(), new ArrayList<>());
+ Row r1 = row.addReplica("c1", "s1");
+ Row r2 = r1.addReplica("c1", "s1");
+ assertEquals(1, r1.collectionVsShardVsReplicas.get("c1").get("s1").size());
+ assertEquals(2, r2.collectionVsShardVsReplicas.get("c1").get("s1").size());
+ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof Policy.ReplicaInfo);
+ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof Policy.ReplicaInfo);
+ }
+
+ public void testMerge() {
+
+ Map map = (Map) Utils.fromJSONString("{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 50}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, 'nodeRole': 'overseer'}," +
+ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}" +
+ " ]," +
+ " 'policies': {" +
+ " 'policy1': [" +
+ " { 'replica': '1', 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +
+ " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'}," +
+ " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" +
+ " ]" +
+ " }" +
+ "}");
+ Policy policy = new Policy(map);
+ List<Clause> clauses = Policy.mergePolicies("mycoll", policy.getPolicies().get("policy1"), policy.getClusterPolicy());
+ Collections.sort(clauses);
+ assertEquals(clauses.size(), 4);
+ assertEquals("1", String.valueOf(clauses.get(0).original.get("replica")));
+ assertEquals("0", String.valueOf(clauses.get(1).original.get("replica")));
+ assertEquals("#ANY", clauses.get(3).original.get("shard"));
+ assertEquals("rack1", clauses.get(2).original.get("sysprop.rack"));
+ assertEquals("overseer", clauses.get(1).original.get("nodeRole"));
+ }
+
+ public void testConditionsSort() {
+ String rules = "{" +
+ " 'cluster-policy':[" +
+ " { 'nodeRole':'overseer', replica: 0, 'strict':false}," +
+ " { 'replica':'<1', 'node':'node3', 'shard':'#EACH'}," +
+ " { 'replica':'<2', 'node':'#ANY', 'shard':'#EACH'}," +
+ " { 'replica':1, 'sysprop.rack':'rack1'}]" +
+ " }";
+ Policy p = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
+ List<Clause> clauses = new ArrayList<>(p.getClusterPolicy());
+ Collections.sort(clauses);
+ assertEquals("nodeRole", clauses.get(1).tag.getName());
+ assertEquals("sysprop.rack", clauses.get(0).tag.getName());
+ }
+
+ public void testRules() throws IOException {
+ String rules = "{" +
+ "cluster-policy:[" +
+ "{nodeRole:'overseer',replica : 0 , strict:false}," +
+ "{replica:'<1',node:node3}," +
+ "{replica:'<2',node:'#ANY', shard:'#EACH'}]," +
+ " cluster-preferences:[" +
+ "{minimize:cores , precision:2}," +
+ "{maximize:freedisk, precision:50}, " +
+ "{minimize:heapUsage, precision:1000}]}";
+
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
+ "}");
+
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
+ Policy.Session session;
+ session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
+
+ List<Row> l = session.getSorted();
+ assertEquals("node1", l.get(0).node);
+ assertEquals("node4", l.get(1).node);
+ assertEquals("node3", l.get(2).node);
+ assertEquals("node2", l.get(3).node);
+
+
+ List<Violation> violations = session.getViolations();
+ assertEquals(3, violations.size());
+ assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.getValue())));
+ assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.getName())));
+ assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.getOperand() == Operand.LESS_THAN && "node".equals(violation.getClause().tag.getName()))));
+
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "gettingstarted")
+ .hint(Hint.SHARD, "r1");
+ SolrParams operation = suggester.getOperation().getParams();
+ assertEquals("node2", operation.get("node"));
+
+ nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
+ "node5:{cores:0, freedisk: 895, heapUsage:17834}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
+ "}");
+ session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
+ SolrRequest opReq = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.TARGET_NODE, "node5")
+ .getOperation();
+ assertNotNull(opReq);
+ assertEquals("node5", opReq.getParams().get("targetNode"));
+
+
+ }
+
+ public void testNegativeConditions() {
+ String autoscaleJson = "{" +
+ " 'cluster-policy':[" +
+ " {'replica':'<4','shard':'#EACH','node':'#ANY'}," +
+ " { 'replica': 0, 'sysprop.fs': '!ssd', 'shard': '#EACH'}," +//negative greedy condition
+ " {'nodeRole':'overseer','replica':'0'}]," +
+ " 'cluster-preferences':[" +
+ " {'minimize':'cores', 'precision':3}," +
+ " {'maximize':'freedisk','precision':100}]}";
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "}");
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
+ ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
+ Policy.Session session = policy.createSession(clusterDataProvider);
+ for (int i = 0; i < 3; i++) {
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
+ SolrRequest op = suggester
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1")
+ .getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+ session = suggester.getSession();
+ }
+
+ }
+
+ public void testGreedyConditions() {
+ String autoscaleJson = "{" +
+ " 'cluster-policy':[" +
+ " {'cores':'<10','node':'#ANY'}," +
+ " {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
+ " { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +//greedy condition
+ " {'nodeRole':'overseer','replica':'0'}]," +
+ " 'cluster-preferences':[" +
+ " {'minimize':'cores', 'precision':3}," +
+ " {'maximize':'freedisk','precision':100}]}";
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "}");
+
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
+ ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
+ ClusterDataProvider cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return clusterDataProvider.getNodeValues(node, tags);
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return clusterDataProvider.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return clusterDataProvider.getNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+ };
+ Policy.Session session = policy.createSession(cdp);
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
+ SolrRequest op = suggester
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1")
+ .getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+ suggester = suggester
+ .getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+
+ suggester = suggester
+ .getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals("node2", op.getParams().get("node"));
+ }
+
+ public void testMoveReplica() {
+ String autoscaleJson = "{" +
+ " 'cluster-policy':[" +
+ " {'cores':'<10','node':'#ANY'}," +
+ " {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
+ " {'nodeRole':'overseer','replica':'0'}]," +
+ " 'cluster-preferences':[" +
+ " {'minimize':'cores', 'precision':3}," +
+ " {'maximize':'freedisk','precision':100}]}";
+
+
+ Map replicaInfoMap = (Map) Utils.fromJSONString("{ '127.0.0.1:60099_solr':{}," +
+ " '127.0.0.1:60089_solr':{'compute_plan_action_test':{'shard1':[" +
+ " {'core_node1':{}}," +
+ " {'core_node2':{}}]}}}");
+ Map m = (Map) Utils.getObjectByPath(replicaInfoMap, false, "127.0.0.1:60089_solr/compute_plan_action_test");
+ m.put("shard1", Arrays.asList(
+ new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Collections.emptyMap()),
+ new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Collections.emptyMap())
+ ));
+
+ Map<String, Map<String, Object>> tagsMap = (Map) Utils.fromJSONString("{" +
+ " '127.0.0.1:60099_solr':{" +
+ " 'cores':0," +
+ " 'freedisk':918005641216}," +
+ " '127.0.0.1:60089_solr':{" +
+ " 'cores':2," +
+ " 'freedisk':918005641216}}}");
+
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
+ Policy.Session session = policy.createSession(new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return tagsMap.get(node);
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return (Map<String, Map<String, List<Policy.ReplicaInfo>>>) replicaInfoMap.get(node);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return replicaInfoMap.keySet();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+ });
+
+ Policy.Suggester suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA)
+ .hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
+ SolrParams op = suggester.getOperation().getParams();
+ assertNotNull(op);
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA).hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
+ op = suggester.getOperation().getParams();
+ assertNotNull(op);
+ }
+
+ public void testOtherTag() {
+ String rules = "{" +
+ "'cluster-preferences':[" +
+ "{'minimize':'cores','precision':2}," +
+ "{'maximize':'freedisk','precision':50}," +
+ "{'minimize':'heapUsage','precision':1000}" +
+ "]," +
+ "'cluster-policy':[" +
+ "{replica:0, 'nodeRole':'overseer','strict':false}," +
+ "{'replica':'<1','node':'node3'}," +
+ "{'replica':'<2','node':'#ANY','shard':'#EACH'}" +
+ "]," +
+ "'policies':{" +
+ "'p1':[" +
+ "{replica:0, 'nodeRole':'overseer','strict':false}," +
+ "{'replica':'<1','node':'node3'}," +
+ "{'replica':'<2','node':'#ANY','shard':'#EACH'}," +
+ "{'replica':'<3','shard':'#EACH','sysprop.rack':'#ANY'}" +
+ "]" +
+ "}" +
+ "}";
+
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, sysprop.rack: rack1}" +
+ "}");
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
+ ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
+ ClusterDataProvider cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return clusterDataProvider.getNodeValues(node, tags);
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return clusterDataProvider.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return clusterDataProvider.getNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return "p1";
+ }
+ };
+ Policy.Session session = policy.createSession(cdp);
+
+ CollectionAdminRequest.AddReplica op = (CollectionAdminRequest.AddReplica) session
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "s1").getOperation();
+ assertNotNull(op);
+ assertEquals("node2", op.getNode());
+ }
+
+ private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
+ return new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return nodeValues.keySet();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return getReplicaDetails(node, clusterState);
+ }
+
+ };
+ }
+ public void testEmptyClusterState(){
+ String autoScaleJson = " {'policies':{'c1':[{" +
+ " 'replica':1," +
+ " 'shard':'#EACH'," +
+ " 'port':'50096'}]}}";
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ " '127.0.0.1:50097_solr':{" +
+ " 'cores':0," +
+ " 'port':'50097'}," +
+ " '127.0.0.1:50096_solr':{" +
+ " 'cores':0," +
+ " 'port':'50096'}}");
+ ClusterDataProvider dataProvider = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return getReplicaDetails(node, clusterState);
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return Arrays.asList( "127.0.0.1:50097_solr", "127.0.0.1:50096_solr");
+ }
+ };
+ Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
+ "newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
+ dataProvider, Collections.singletonMap("newColl", "c1"), Arrays.asList("shard1", "shard2"), 1, null);
+ assertTrue(locations.get("shard1").containsAll(ImmutableList.of("127.0.0.1:50096_solr")));
+ assertTrue(locations.get("shard2").containsAll(ImmutableList.of("127.0.0.1:50096_solr")));
+ }
+
+ public void testMultiReplicaPlacement() {
+ String autoScaleJson = "{" +
+ " 'cluster-preferences': [" +
+ " { maximize : freedisk , precision: 50}," +
+ " { minimize : cores, precision: 2}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { replica : '0' , 'nodeRole': 'overseer'}," +
+ " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'" +
+ " }" +
+ " ]," +
+ " 'policies': {" +
+ " 'policy1': [" +
+ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" +
+ " ]" +
+ " }" +
+ "}";
+
+
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heap:10480, sysprop.rack:rack3}," +
+ "node2:{cores:4, freedisk: 749, heap:6873, sysprop.fs : ssd, sysprop.rack:rack1}," +
+ "node3:{cores:7, freedisk: 262, heap:7834, sysprop.rack:rack4}," +
+ "node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, sysprop.rack:rack2}" +
+ "}");
+
+ ClusterDataProvider dataProvider = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return getReplicaDetails(node, clusterState);
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return Arrays.asList("node1", "node2", "node3", "node4");
+ }
+ };
+ Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
+ "newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
+ dataProvider, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3, null);
+ assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2", "node1", "node3")));
+ assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2", "node1", "node3")));
+
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b99ee2b1/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
deleted file mode 100644
index a036c8d..0000000
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ /dev/null
@@ -1,661 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.autoscaling.Cell;
-import org.apache.solr.client.solrj.cloud.autoscaling.Clause;
-import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
-import org.apache.solr.client.solrj.cloud.autoscaling.Operand;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
-import org.apache.solr.client.solrj.cloud.autoscaling.Row;
-import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.common.util.ValidatingJsonMap;
-
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
-
-public class TestPolicy extends SolrTestCaseJ4 {
-
- public static String clusterState = "{'gettingstarted':{" +
- " 'router':{'name':'compositeId'}," +
- " 'shards':{" +
- " 'shard1':{" +
- " 'range':'80000000-ffffffff'," +
- " 'replicas':{" +
- " 'r1':{" +
- " 'core':r1," +
- " 'base_url':'http://10.0.0.4:8983/solr'," +
- " 'node_name':'node1'," +
- " 'state':'active'," +
- " 'leader':'true'}," +
- " 'r2':{" +
- " 'core':r2," +
- " 'base_url':'http://10.0.0.4:7574/solr'," +
- " 'node_name':'node2'," +
- " 'state':'active'}}}," +
- " 'shard2':{" +
- " 'range':'0-7fffffff'," +
- " 'replicas':{" +
- " 'r3':{" +
- " 'core':r3," +
- " 'base_url':'http://10.0.0.4:8983/solr'," +
- " 'node_name':'node1'," +
- " 'state':'active'," +
- " 'leader':'true'}," +
- " 'r4':{" +
- " 'core':r4," +
- " 'base_url':'http://10.0.0.4:8987/solr'," +
- " 'node_name':'node4'," +
- " 'state':'active'}," +
- " 'r6':{" +
- " 'core':r6," +
- " 'base_url':'http://10.0.0.4:8989/solr'," +
- " 'node_name':'node3'," +
- " 'state':'active'}," +
- " 'r5':{" +
- " 'core':r5," +
- " 'base_url':'http://10.0.0.4:7574/solr'," +
- " 'node_name':'node1'," +
- " 'state':'active'}}}}}}";
-
- public static Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaDetails(String node, String s) {
- ValidatingJsonMap m = ValidatingJsonMap
- .getDeepCopy((Map) Utils.fromJSONString(s), 6, true);
- Map<String, Map<String, List<Policy.ReplicaInfo>>> result = new LinkedHashMap<>();
-
- m.forEach((collName, o) -> {
- ValidatingJsonMap coll = (ValidatingJsonMap) o;
- coll.getMap("shards").forEach((shard, o1) -> {
- ValidatingJsonMap sh = (ValidatingJsonMap) o1;
- sh.getMap("replicas").forEach((replicaName, o2) -> {
- ValidatingJsonMap r = (ValidatingJsonMap) o2;
- String node_name = (String) r.get("node_name");
- if (!node_name.equals(node)) return;
- Map<String, List<Policy.ReplicaInfo>> shardVsReplicaStats = result.get(collName);
- if (shardVsReplicaStats == null) result.put(collName, shardVsReplicaStats = new HashMap<>());
- List<Policy.ReplicaInfo> replicaInfos = shardVsReplicaStats.get(shard);
- if (replicaInfos == null) shardVsReplicaStats.put(shard, replicaInfos = new ArrayList<>());
- replicaInfos.add(new Policy.ReplicaInfo(replicaName, collName, shard, new HashMap<>()));
- });
- });
- });
- return result;
- }
-
- public void testValidate() {
- expectError("replica", -1, "must be greater than" );
- expectError("replica","hello", "not a valid number" );
- assertEquals( 1l, Clause.validate("replica", "1", true));
- assertEquals("c", Clause.validate("collection", "c", true));
- assertEquals( "s", Clause.validate("shard", "s",true));
- assertEquals( "overseer", Clause.validate("nodeRole", "overseer",true));
-
- expectError("nodeRole", "wrong","must be one of");
-
- expectError("sysLoadAvg", "101","must be less than ");
- expectError("sysLoadAvg", 101,"must be less than ");
- expectError("sysLoadAvg", "-1","must be greater than");
- expectError("sysLoadAvg", -1,"must be greater than");
-
- assertEquals(12.46d,Clause.validate("sysLoadAvg", "12.46",true));
- assertEquals(12.46,Clause.validate("sysLoadAvg", 12.46d,true));
-
-
- expectError("ip_1", "300","must be less than ");
- expectError("ip_1", 300,"must be less than ");
- expectError("ip_1", "-1","must be greater than");
- expectError("ip_1", -1,"must be greater than");
-
- assertEquals(1l,Clause.validate("ip_1", "1",true));
-
- expectError("heapUsage", "-1","must be greater than");
- expectError("heapUsage", -1,"must be greater than");
- assertEquals(69.9d,Clause.validate("heapUsage", "69.9",true));
- assertEquals(69.9d,Clause.validate("heapUsage", 69.9d,true));
-
- expectError("port", "70000","must be less than ");
- expectError("port", 70000,"must be less than ");
- expectError("port", "0","must be greater than");
- expectError("port", 0,"must be greater than");
-
- expectError("cores", "-1","must be greater than");
-
-
- }
-
- private static void expectError(String name, Object val, String msg){
- try {
- Clause.validate(name, val,true);
- fail("expected exception containing "+msg);
- } catch (Exception e) {
- assertTrue("expected exception containing "+msg,e.getMessage().contains(msg));
- }
-
- }
-
- public void testOperands() {
- Clause c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:'<2', node:'#ANY'}"));
- assertFalse(c.replica.isPass(3));
- assertFalse(c.replica.isPass(2));
- assertTrue(c.replica.isPass(1));
-
- c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:'>2', node:'#ANY'}"));
- assertTrue(c.replica.isPass(3));
- assertFalse(c.replica.isPass(2));
- assertFalse(c.replica.isPass(1));
-
- c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}"));
- assertTrue(c.tag.isPass("OVERSEER"));
- assertFalse(c.tag.isPass("overseer"));
-
- c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'<12.7'}"));
- assertTrue(c.tag.isPass("12.6"));
- assertTrue(c.tag.isPass(12.6d));
- assertFalse(c.tag.isPass("12.9"));
- assertFalse(c.tag.isPass(12.9d));
-
- c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'>12.7'}"));
- assertTrue(c.tag.isPass("12.8"));
- assertTrue(c.tag.isPass(12.8d));
- assertFalse(c.tag.isPass("12.6"));
- assertFalse(c.tag.isPass(12.6d));
- }
-
- public void testRow() {
- Row row = new Row("nodex", new Cell[]{new Cell(0, "node", "nodex")}, false, new HashMap<>(), new ArrayList<>());
- Row r1 = row.addReplica("c1", "s1");
- Row r2 = r1.addReplica("c1", "s1");
- assertEquals(1, r1.collectionVsShardVsReplicas.get("c1").get("s1").size());
- assertEquals(2, r2.collectionVsShardVsReplicas.get("c1").get("s1").size());
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof Policy.ReplicaInfo);
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof Policy.ReplicaInfo);
- }
-
- public void testMerge() {
-
- Map map = (Map) Utils.fromJSONString("{" +
- " 'cluster-preferences': [" +
- " { 'maximize': 'freedisk', 'precision': 50}," +
- " { 'minimize': 'cores', 'precision': 50}" +
- " ]," +
- " 'cluster-policy': [" +
- " { 'replica': 0, 'nodeRole': 'overseer'}," +
- " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]," +
- " 'policies': {" +
- " 'policy1': [" +
- " { 'replica': '1', 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +
- " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'}," +
- " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" +
- " ]" +
- " }" +
- "}");
- Policy policy = new Policy(map);
- List<Clause> clauses = Policy.mergePolicies("mycoll", policy.getPolicies().get("policy1"), policy.getClusterPolicy());
- Collections.sort(clauses);
- assertEquals(clauses.size(), 4);
- assertEquals("1", String.valueOf(clauses.get(0).original.get("replica")));
- assertEquals("0", String.valueOf(clauses.get(1).original.get("replica")));
- assertEquals("#ANY", clauses.get(3).original.get("shard"));
- assertEquals("rack1", clauses.get(2).original.get("sysprop.rack"));
- assertEquals("overseer", clauses.get(1).original.get("nodeRole"));
- }
-
- public void testConditionsSort() {
- String rules = "{" +
- " 'cluster-policy':[" +
- " { 'nodeRole':'overseer', replica: 0, 'strict':false}," +
- " { 'replica':'<1', 'node':'node3', 'shard':'#EACH'}," +
- " { 'replica':'<2', 'node':'#ANY', 'shard':'#EACH'}," +
- " { 'replica':1, 'sysprop.rack':'rack1'}]" +
- " }";
- Policy p = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
- List<Clause> clauses = new ArrayList<>(p.getClusterPolicy());
- Collections.sort(clauses);
- assertEquals("nodeRole", clauses.get(1).tag.getName());
- assertEquals("sysprop.rack", clauses.get(0).tag.getName());
- }
-
- public void testRules() throws IOException {
- String rules = "{" +
- "cluster-policy:[" +
- "{nodeRole:'overseer',replica : 0 , strict:false}," +
- "{replica:'<1',node:node3}," +
- "{replica:'<2',node:'#ANY', shard:'#EACH'}]," +
- " cluster-preferences:[" +
- "{minimize:cores , precision:2}," +
- "{maximize:freedisk, precision:50}, " +
- "{minimize:heapUsage, precision:1000}]}";
-
- Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
- "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
- "}");
-
- Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
- Policy.Session session;
- session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
-
- List<Row> l = session.getSorted();
- assertEquals("node1", l.get(0).node);
- assertEquals("node4", l.get(1).node);
- assertEquals("node3", l.get(2).node);
- assertEquals("node2", l.get(3).node);
-
-
- List<Violation> violations = session.getViolations();
- assertEquals(3, violations.size());
- assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.getValue())));
- assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.getName())));
- assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.getOperand() == Operand.LESS_THAN && "node".equals(violation.getClause().tag.getName()))));
-
- Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
- .hint(Hint.COLL, "gettingstarted")
- .hint(Hint.SHARD, "r1");
- SolrParams operation = suggester.getOperation().getParams();
- assertEquals("node2", operation.get("node"));
-
- nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
- "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
- "node5:{cores:0, freedisk: 895, heapUsage:17834}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
- "}");
- session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
- SolrRequest opReq = session.getSuggester(MOVEREPLICA)
- .hint(Hint.TARGET_NODE, "node5")
- .getOperation();
- assertNotNull(opReq);
- assertEquals("node5", opReq.getParams().get("targetNode"));
-
-
- }
-
- public void testNegativeConditions() {
- String autoscaleJson = "{" +
- " 'cluster-policy':[" +
- " {'replica':'<4','shard':'#EACH','node':'#ANY'}," +
- " { 'replica': 0, 'sysprop.fs': '!ssd', 'shard': '#EACH'}," +//negative greedy condition
- " {'nodeRole':'overseer','replica':'0'}]," +
- " 'cluster-preferences':[" +
- " {'minimize':'cores', 'precision':3}," +
- " {'maximize':'freedisk','precision':100}]}";
- Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
- "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
- "}");
- Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
- ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
- Policy.Session session = policy.createSession(clusterDataProvider);
- for (int i = 0; i < 3; i++) {
- Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
- SolrRequest op = suggester
- .hint(Hint.COLL, "newColl")
- .hint(Hint.SHARD, "shard1")
- .getOperation();
- assertNotNull(op);
- assertEquals("node3", op.getParams().get("node"));
- session = suggester.getSession();
- }
-
- }
-
- public void testGreedyConditions() {
- String autoscaleJson = "{" +
- " 'cluster-policy':[" +
- " {'cores':'<10','node':'#ANY'}," +
- " {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
- " { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +//greedy condition
- " {'nodeRole':'overseer','replica':'0'}]," +
- " 'cluster-preferences':[" +
- " {'minimize':'cores', 'precision':3}," +
- " {'maximize':'freedisk','precision':100}]}";
- Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
- "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
- "}");
-
- Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
- ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
- ClusterDataProvider cdp = new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return clusterDataProvider.getNodeValues(node, tags);
- }
-
- @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return clusterDataProvider.getReplicaInfo(node, keys);
- }
-
- @Override
- public Collection<String> getNodes() {
- return clusterDataProvider.getNodes();
- }
-
- @Override
- public String getPolicyNameByCollection(String coll) {
- return null;
- }
- };
- Policy.Session session = policy.createSession(cdp);
- Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
- SolrRequest op = suggester
- .hint(Hint.COLL, "newColl")
- .hint(Hint.SHARD, "shard1")
- .getOperation();
- assertNotNull(op);
- assertEquals("node3", op.getParams().get("node"));
- suggester = suggester
- .getSession()
- .getSuggester(ADDREPLICA)
- .hint(Hint.COLL, "newColl")
- .hint(Hint.SHARD, "shard1");
- op = suggester.getOperation();
- assertNotNull(op);
- assertEquals("node3", op.getParams().get("node"));
-
- suggester = suggester
- .getSession()
- .getSuggester(ADDREPLICA)
- .hint(Hint.COLL, "newColl")
- .hint(Hint.SHARD, "shard1");
- op = suggester.getOperation();
- assertNotNull(op);
- assertEquals("node2", op.getParams().get("node"));
- }
-
- public void testMoveReplica() {
- String autoscaleJson = "{" +
- " 'cluster-policy':[" +
- " {'cores':'<10','node':'#ANY'}," +
- " {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
- " {'nodeRole':'overseer','replica':'0'}]," +
- " 'cluster-preferences':[" +
- " {'minimize':'cores', 'precision':3}," +
- " {'maximize':'freedisk','precision':100}]}";
-
-
- Map replicaInfoMap = (Map) Utils.fromJSONString("{ '127.0.0.1:60099_solr':{}," +
- " '127.0.0.1:60089_solr':{'compute_plan_action_test':{'shard1':[" +
- " {'core_node1':{}}," +
- " {'core_node2':{}}]}}}");
- Map m = (Map) Utils.getObjectByPath(replicaInfoMap, false, "127.0.0.1:60089_solr/compute_plan_action_test");
- m.put("shard1", Arrays.asList(
- new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Collections.emptyMap()),
- new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Collections.emptyMap())
- ));
-
- Map<String, Map<String, Object>> tagsMap = (Map) Utils.fromJSONString("{" +
- " '127.0.0.1:60099_solr':{" +
- " 'cores':0," +
- " 'freedisk':918005641216}," +
- " '127.0.0.1:60089_solr':{" +
- " 'cores':2," +
- " 'freedisk':918005641216}}}");
-
- Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
- Policy.Session session = policy.createSession(new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return tagsMap.get(node);
- }
-
- @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return (Map<String, Map<String, List<Policy.ReplicaInfo>>>) replicaInfoMap.get(node);
- }
-
- @Override
- public Collection<String> getNodes() {
- return replicaInfoMap.keySet();
- }
-
- @Override
- public String getPolicyNameByCollection(String coll) {
- return null;
- }
- });
-
- Policy.Suggester suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA)
- .hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
- SolrParams op = suggester.getOperation().getParams();
- assertNotNull(op);
- session = suggester.getSession();
- suggester = session.getSuggester(MOVEREPLICA).hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
- op = suggester.getOperation().getParams();
- assertNotNull(op);
- }
-
- public void testOtherTag() {
- String rules = "{" +
- "'cluster-preferences':[" +
- "{'minimize':'cores','precision':2}," +
- "{'maximize':'freedisk','precision':50}," +
- "{'minimize':'heapUsage','precision':1000}" +
- "]," +
- "'cluster-policy':[" +
- "{replica:0, 'nodeRole':'overseer','strict':false}," +
- "{'replica':'<1','node':'node3'}," +
- "{'replica':'<2','node':'#ANY','shard':'#EACH'}" +
- "]," +
- "'policies':{" +
- "'p1':[" +
- "{replica:0, 'nodeRole':'overseer','strict':false}," +
- "{'replica':'<1','node':'node3'}," +
- "{'replica':'<2','node':'#ANY','shard':'#EACH'}," +
- "{'replica':'<3','shard':'#EACH','sysprop.rack':'#ANY'}" +
- "]" +
- "}" +
- "}";
-
- Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
- "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
- "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2}," +
- "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, sysprop.rack: rack1}" +
- "}");
- Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
- ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
- ClusterDataProvider cdp = new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return clusterDataProvider.getNodeValues(node, tags);
- }
-
- @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return clusterDataProvider.getReplicaInfo(node, keys);
- }
-
- @Override
- public Collection<String> getNodes() {
- return clusterDataProvider.getNodes();
- }
-
- @Override
- public String getPolicyNameByCollection(String coll) {
- return "p1";
- }
- };
- Policy.Session session = policy.createSession(cdp);
-
- CollectionAdminRequest.AddReplica op = (CollectionAdminRequest.AddReplica) session
- .getSuggester(ADDREPLICA)
- .hint(Hint.COLL, "newColl")
- .hint(Hint.SHARD, "s1").getOperation();
- assertNotNull(op);
- assertEquals("node2", op.getNode());
- }
-
- private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
- return new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- Map<String, Object> result = new LinkedHashMap<>();
- tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
- return result;
- }
-
- @Override
- public Collection<String> getNodes() {
- return nodeValues.keySet();
- }
-
- @Override
- public String getPolicyNameByCollection(String coll) {
- return null;
- }
-
- @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return getReplicaDetails(node, clusterState);
- }
-
- };
- }
- public void testEmptyClusterState(){
- String autoScaleJson = " {'policies':{'c1':[{" +
- " 'replica':1," +
- " 'shard':'#EACH'," +
- " 'port':'50096'}]}}";
- Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- " '127.0.0.1:50097_solr':{" +
- " 'cores':0," +
- " 'port':'50097'}," +
- " '127.0.0.1:50096_solr':{" +
- " 'cores':0," +
- " 'port':'50096'}}");
- ClusterDataProvider dataProvider = new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
- Map<String, Object> result = new LinkedHashMap<>();
- keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
- return result;
- }
-
- @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return getReplicaDetails(node, clusterState);
- }
-
- @Override
- public String getPolicyNameByCollection(String coll) {
- return null;
- }
-
- @Override
- public Collection<String> getNodes() {
- return Arrays.asList( "127.0.0.1:50097_solr", "127.0.0.1:50096_solr");
- }
- };
- Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
- "newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
- dataProvider, Collections.singletonMap("newColl", "c1"), Arrays.asList("shard1", "shard2"), 1, null);
- assertTrue(locations.get("shard1").containsAll(ImmutableList.of("127.0.0.1:50096_solr")));
- assertTrue(locations.get("shard2").containsAll(ImmutableList.of("127.0.0.1:50096_solr")));
- }
-
- public void testMultiReplicaPlacement() {
- String autoScaleJson = "{" +
- " 'cluster-preferences': [" +
- " { maximize : freedisk , precision: 50}," +
- " { minimize : cores, precision: 2}" +
- " ]," +
- " 'cluster-policy': [" +
- " { replica : '0' , 'nodeRole': 'overseer'}," +
- " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'" +
- " }" +
- " ]," +
- " 'policies': {" +
- " 'policy1': [" +
- " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" +
- " ]" +
- " }" +
- "}";
-
-
- Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heap:10480, sysprop.rack:rack3}," +
- "node2:{cores:4, freedisk: 749, heap:6873, sysprop.fs : ssd, sysprop.rack:rack1}," +
- "node3:{cores:7, freedisk: 262, heap:7834, sysprop.rack:rack4}," +
- "node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, sysprop.rack:rack2}" +
- "}");
-
- ClusterDataProvider dataProvider = new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
- Map<String, Object> result = new LinkedHashMap<>();
- keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
- return result;
- }
-
- @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return getReplicaDetails(node, clusterState);
- }
-
- @Override
- public String getPolicyNameByCollection(String coll) {
- return null;
- }
-
- @Override
- public Collection<String> getNodes() {
- return Arrays.asList("node1", "node2", "node3", "node4");
- }
- };
- Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
- "newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
- dataProvider, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3, null);
- assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2", "node1", "node3")));
- assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2", "node1", "node3")));
-
-
- }
-
-
-}
[13/18] lucene-solr:feature/autoscaling: SOLR-10931: Refactoring the
package name for autoscaling client classes
Posted by sh...@apache.org.
SOLR-10931: Refactoring the package name for autoscaling client classes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/13a3ae29
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/13a3ae29
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/13a3ae29
Branch: refs/heads/feature/autoscaling
Commit: 13a3ae292277d770cb5219142729c14db9744fa5
Parents: ce4a791
Author: Ishan Chattopadhyaya <is...@lucidworks.com>
Authored: Wed Jun 28 07:55:40 2017 +0530
Committer: Ishan Chattopadhyaya <is...@lucidworks.com>
Committed: Wed Jun 28 07:55:40 2017 +0530
----------------------------------------------------------------------
.../src/java/org/apache/solr/cloud/Assign.java | 6 +-
.../cloud/OverseerCollectionMessageHandler.java | 4 +-
.../cloud/autoscaling/AutoScalingHandler.java | 13 +-
.../solr/handler/admin/CollectionsHandler.java | 2 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 69 +++
.../client/solrj/cloud/autoscaling/Cell.java | 69 +++
.../client/solrj/cloud/autoscaling/Clause.java | 472 +++++++++++++++++
.../cloud/autoscaling/ClusterDataProvider.java | 52 ++
.../cloud/autoscaling/MoveReplicaSuggester.java | 83 +++
.../client/solrj/cloud/autoscaling/Operand.java | 123 +++++
.../client/solrj/cloud/autoscaling/Policy.java | 521 +++++++++++++++++++
.../solrj/cloud/autoscaling/PolicyHelper.java | 96 ++++
.../solrj/cloud/autoscaling/Preference.java | 89 ++++
.../client/solrj/cloud/autoscaling/Row.java | 120 +++++
.../solrj/cloud/autoscaling/package-info.java | 23 +
.../solrj/impl/SolrClientDataProvider.java | 4 +-
.../solrj/request/CollectionAdminRequest.java | 2 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 69 ---
.../org/apache/solr/cloud/autoscaling/Cell.java | 57 --
.../apache/solr/cloud/autoscaling/Clause.java | 460 ----------------
.../cloud/autoscaling/ClusterDataProvider.java | 52 --
.../cloud/autoscaling/MoveReplicaSuggester.java | 83 ---
.../apache/solr/cloud/autoscaling/Operand.java | 123 -----
.../apache/solr/cloud/autoscaling/Policy.java | 519 ------------------
.../solr/cloud/autoscaling/PolicyHelper.java | 96 ----
.../solr/cloud/autoscaling/Preference.java | 85 ---
.../org/apache/solr/cloud/autoscaling/Row.java | 116 -----
.../solr/cloud/autoscaling/package-info.java | 23 -
.../apache/solr/common/cloud/DocCollection.java | 2 +-
.../solr/cloud/autoscaling/TestPolicy.java | 23 +-
30 files changed, 1751 insertions(+), 1705 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/cloud/Assign.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Assign.java b/solr/core/src/java/org/apache/solr/cloud/Assign.java
index cdcf415..9f21245 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Assign.java
@@ -29,11 +29,11 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
-import org.apache.solr.cloud.autoscaling.Policy;
-import org.apache.solr.cloud.autoscaling.PolicyHelper;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
@@ -48,7 +48,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.KeeperException;
import static java.util.Collections.singletonMap;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index a055033..c6051d7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -37,12 +37,12 @@ import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.cloud.autoscaling.Policy;
import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.ReplicaAssigner.Position;
@@ -81,7 +81,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 5946bf4..356ce37 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -28,6 +28,11 @@ import java.util.Map;
import com.google.common.collect.ImmutableSet;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.cloud.autoscaling.Cell;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.cloud.autoscaling.Preference;
+import org.apache.solr.client.solrj.cloud.autoscaling.Row;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.common.SolrException;
@@ -139,11 +144,11 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
List<Map<String, Object>> sortedNodes = new ArrayList<>(sorted.size());
for (Row row : sorted) {
Map<String, Object> map = Utils.makeMap("node", row.node);
- for (Cell cell : row.cells) {
+ for (Cell cell : row.getCells()) {
for (Preference clusterPreference : clusterPreferences) {
- Policy.SortParam name = clusterPreference.name;
- if (cell.name.equalsIgnoreCase(name.name())) {
- map.put(name.name(), cell.val);
+ Policy.SortParam name = clusterPreference.getName();
+ if (cell.getName().equalsIgnoreCase(name.name())) {
+ map.put(name.name(), cell.getValue());
break;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 3f2eb5f..122aa4a 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -92,6 +92,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.client.solrj.response.RequestStatusState.COMPLETED;
import static org.apache.solr.client.solrj.response.RequestStatusState.FAILED;
import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND;
@@ -109,7 +110,6 @@ import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_IF_DOW
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.REQUESTID;
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP;
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARD_UNIQUE;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
import static org.apache.solr.common.cloud.DocCollection.RULE;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
new file mode 100644
index 0000000..01149f3
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.List;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+
+class AddReplicaSuggester extends Suggester {
+
+ SolrRequest init() {
+ SolrRequest operation = tryEachNode(true);
+ if (operation == null) operation = tryEachNode(false);
+ return operation;
+ }
+
+ SolrRequest tryEachNode(boolean strict) {
+ String coll = (String) hints.get(Hint.COLL);
+ String shard = (String) hints.get(Hint.SHARD);
+ if (coll == null || shard == null)
+ throw new RuntimeException("add-replica requires 'collection' and 'shard'");
+ //iterate through elements and identify the least loaded
+
+ List<Clause.Violation> leastSeriousViolation = null;
+ Integer targetNodeIndex = null;
+ for (int i = getMatrix().size() - 1; i >= 0; i--) {
+ Row row = getMatrix().get(i);
+ if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
+ Row tmpRow = row.addReplica(coll, shard);
+ tmpRow.violations.clear();
+
+ List<Clause.Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
+ if(!containsNewErrors(errs)) {
+ if(isLessSerious(errs, leastSeriousViolation)){
+ leastSeriousViolation = errs;
+ targetNodeIndex = i;
+ }
+ }
+ }
+
+ if (targetNodeIndex != null) {// there are no rule violations
+ getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
+ return CollectionAdminRequest
+ .addReplicaToShard(coll, shard)
+ .setNode(getMatrix().get(targetNodeIndex).node);
+ }
+
+ return null;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
new file mode 100644
index 0000000..0ac2b36
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Utils;
+
+public class Cell implements MapWriter {
+ final int index;
+ final String name;
+ Object val, approxVal;
+
+ public Cell(int index, String name, Object val) {
+ this.index = index;
+ this.name = name;
+ this.val = val;
+ }
+
+ public Cell(int index, String name, Object val, Object approxVal) {
+ this.index = index;
+ this.name = name;
+ this.val = val;
+ this.approxVal = approxVal;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, val);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(this.toMap(new HashMap<>()));
+ }
+
+ public Cell copy() {
+ return new Cell(index, name, val, approxVal);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Object getValue() {
+ return val;
+ }
+
+ public Object getApproxValue() {
+ return approxVal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
new file mode 100644
index 0000000..ff56627
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+import static java.util.Collections.singletonMap;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
+import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
+import static org.apache.solr.common.params.CoreAdminParams.SHARD;
+
+// a set of conditions in a policy
+public class Clause implements MapWriter, Comparable<Clause> {
+ public Map<String, Object> original;
+ public Condition collection, shard, replica, tag, globalTag;
+
+ boolean strict = true;
+
+ public Clause(Map<String, Object> m) {
+ this.original = m;
+ strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true")));
+ Optional<String> globalTagName = m.keySet().stream().filter(Policy.GLOBAL_ONLY_TAGS::contains).findFirst();
+ if (globalTagName.isPresent()) {
+ globalTag = parse(globalTagName.get(), m);
+ if (m.size() > 2) {
+ throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + Utils.toJSONString(m));
+ }
+ tag = parse(m.keySet().stream()
+ .filter(s -> (!globalTagName.get().equals(s) && !IGNORE_TAGS.contains(s)))
+ .findFirst().get(), m);
+ } else {
+ collection = parse(COLLECTION, m);
+ shard = parse(SHARD, m);
+ if(m.get(REPLICA) == null){
+ throw new RuntimeException(StrUtils.formatString("'replica' is required in {0}", Utils.toJSONString(m)));
+ }
+ this.replica = parse(REPLICA, m);
+ if (replica.op == WILDCARD) throw new RuntimeException("replica val cannot be null" + Utils.toJSONString(m));
+ m.forEach((s, o) -> parseCondition(s, o));
+ }
+ if (tag == null)
+ throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
+
+ }
+
+ public boolean doesOverride(Clause that) {
+ return (collection.equals(that.collection) &&
+ tag.name.equals(that.tag.name));
+
+ }
+
+ public boolean isPerCollectiontag() {
+ return globalTag == null;
+ }
+
+ void parseCondition(String s, Object o) {
+ if (IGNORE_TAGS.contains(s)) return;
+ if (tag != null) {
+ throw new IllegalArgumentException("Only one tag other than collection, shard, replica is possible");
+ }
+ tag = parse(s, singletonMap(s, o));
+ }
+
+ @Override
+ public int compareTo(Clause that) {
+ try {
+ int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
+ if (v != 0) return v;
+ if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
+ v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
+ if (v == 0) {
+ v = Long.compare((Long) this.replica.val, (Long) that.replica.val);
+ v = this.replica.op == LESS_THAN ? v : v * -1;
+ }
+ return v;
+ } else {
+ return 0;
+ }
+ } catch (NullPointerException e) {
+ throw e;
+ }
+ }
+
+ void addTags(List<String> params) {
+ if (globalTag != null && !params.contains(globalTag.name)) params.add(globalTag.name);
+ if (tag != null && !params.contains(tag.name)) params.add(tag.name);
+ }
+
+ public static class Condition {
+ final String name;
+ final Object val;
+ final Operand op;
+
+ Condition(String name, Object val, Operand op) {
+ this.name = name;
+ this.val = val;
+ this.op = op;
+ }
+
+ TestStatus match(Row row) {
+ return op.match(val, row.getVal(name));
+ }
+
+ TestStatus match(Object testVal) {
+ return op.match(this.val, testVal);
+ }
+
+ public boolean isPass(Object inputVal) {
+ return op.match(val, validate(name, inputVal, false)) == PASS;
+ }
+
+ public boolean isPass(Row row) {
+ return op.match(val, row.getVal(name)) == PASS;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that instanceof Condition) {
+ Condition c = (Condition) that;
+ return Objects.equals(c.name, name) && Objects.equals(c.val, val) && c.op == op;
+ }
+ return false;
+ }
+
+ public Integer delta(Object val) {
+ return op.delta(this.val, val);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Object getValue() {
+ return val;
+ }
+
+ public Operand getOperand() {
+ return op;
+ }
+ }
+
+ static Condition parse(String s, Map m) {
+ Object expectedVal = null;
+ Object val = m.get(s);
+ try {
+ String conditionName = s.trim();
+ Operand operand = null;
+ if (val == null) {
+ operand = WILDCARD;
+ expectedVal = Policy.ANY;
+ } else if (val instanceof String) {
+ String strVal = ((String) val).trim();
+ if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
+ else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
+ else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
+ else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
+ else operand = EQUAL;
+ expectedVal = validate(s, strVal.substring(EQUAL == operand || WILDCARD == operand ? 0 : 1), true);
+ } else if (val instanceof Number) {
+ operand = EQUAL;
+ expectedVal = validate(s, val, true);
+ }
+ return new Condition(conditionName, expectedVal, operand);
+
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid tag : " + s + ":" + val, e);
+ }
+ }
+
+ public class Violation implements MapWriter {
+ final String shard, coll, node;
+ final Object actualVal;
+ final Integer delta;//how far is the actual value from the expected value
+ final Object tagKey;
+ private final int hash;
+
+
+ private Violation(String coll, String shard, String node, Object actualVal, Integer delta, Object tagKey) {
+ this.shard = shard;
+ this.coll = coll;
+ this.node = node;
+ this.delta = delta;
+ this.actualVal = actualVal;
+ this.tagKey = tagKey;
+ hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
+ }
+
+ public Clause getClause() {
+ return Clause.this;
+ }
+
+ @Override
+ public int hashCode() {
+ return hash;
+ }
+ //if the delta is lower , this violation is less serious
+ public boolean isLessSerious(Violation that) {
+ return that.delta != null && delta != null &&
+ Math.abs(delta) < Math.abs(that.delta);
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that instanceof Violation) {
+ Violation v = (Violation) that;
+ return Objects.equals(this.shard, v.shard) &&
+ Objects.equals(this.coll, v.coll) &&
+ Objects.equals(this.node, v.node) &&
+ Objects.equals(this.tagKey, v.tagKey)
+ ;
+ }
+ return false;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.putIfNotNull("collection", coll);
+ ew.putIfNotNull("shard", shard);
+ ew.putIfNotNull("node", node);
+ ew.putIfNotNull("tagKey", String.valueOf(tagKey));
+ ew.putIfNotNull("violation", (MapWriter) ew1 -> {
+ ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
+ String.valueOf(actualVal));
+ ew1.putIfNotNull("delta", delta);
+ });
+ ew.put("clause", getClause());
+ }
+ }
+
+
+ public List<Violation> test(List<Row> allRows) {
+ List<Violation> violations = new ArrayList<>();
+ if (isPerCollectiontag()) {
+ Map<String, Map<String, Map<String, AtomicInteger>>> replicaCount = computeReplicaCounts(allRows);
+ for (Map.Entry<String, Map<String, Map<String, AtomicInteger>>> e : replicaCount.entrySet()) {
+ if (!collection.isPass(e.getKey())) continue;
+ for (Map.Entry<String, Map<String, AtomicInteger>> shardVsCount : e.getValue().entrySet()) {
+ if (!shard.isPass(shardVsCount.getKey())) continue;
+ for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
+ if (!replica.isPass(counts.getValue())) {
+ violations.add(new Violation(
+ e.getKey(),
+ shardVsCount.getKey(),
+ tag.name.equals("node") ? counts.getKey() : null,
+ counts.getValue(),
+ replica.delta(counts.getValue()),
+ counts.getKey()
+ ));
+ }
+ }
+ }
+ }
+ } else {
+ for (Row r : allRows) {
+ if (!tag.isPass(r)) {
+ violations.add(new Violation(null, null, r.node, r.getVal(tag.name), tag.delta(r.getVal(tag.name)), null));
+ }
+ }
+ }
+ return violations;
+
+ }
+
+
+ private Map<String, Map<String, Map<String, AtomicInteger>>> computeReplicaCounts(List<Row> allRows) {
+ Map<String, Map<String, Map<String, AtomicInteger>>> collVsShardVsTagVsCount = new HashMap<>();
+ for (Row row : allRows)
+ for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
+ String collectionName = colls.getKey();
+ if (!collection.isPass(collectionName)) continue;
+ collVsShardVsTagVsCount.putIfAbsent(collectionName, new HashMap<>());
+ Map<String, Map<String, AtomicInteger>> collMap = collVsShardVsTagVsCount.get(collectionName);
+ for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
+ String shardName = shards.getKey();
+ if (ANY.equals(shard.val)) shardName = ANY;
+ if (!shard.isPass(shardName)) break;
+ collMap.putIfAbsent(shardName, new HashMap<>());
+ Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
+ Object tagVal = row.getVal(tag.name);
+ tagVsCount.putIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", new AtomicInteger());
+ if (tag.isPass(tagVal)) {
+ tagVsCount.get(String.valueOf(tagVal)).addAndGet(shards.getValue().size());
+ }
+ }
+ }
+ return collVsShardVsTagVsCount;
+ }
+
+ public boolean isStrict() {
+ return strict;
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(original);
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (Map.Entry<String, Object> e : original.entrySet()) ew.put(e.getKey(), e.getValue());
+ }
+
+ enum TestStatus {
+ NOT_APPLICABLE, FAIL, PASS
+ }
+
+ private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict"));
+
+ static class ValidateInfo {
+ final Class type;
+ final Set<String> vals;
+ final Number min;
+ final Number max;
+
+
+ ValidateInfo(Class type, Set<String> vals, Number min, Number max) {
+ this.type = type;
+ this.vals = vals;
+ this.min = min;
+ if(min != null && !type.isInstance(min)) throw new RuntimeException("wrong min value type, expected: " + type.getName() + " actual: " + min.getClass().getName());
+ this.max = max;
+ if(max != null && !type.isInstance(max)) throw new RuntimeException("wrong max value type, expected: " + type.getName() + " actual: " + max.getClass().getName());
+ }
+ }
+
+
+ /**
+ *
+ * @param name name of the condition
+ * @param val value of the condition
+ * @param isRuleVal is this provided in the rule
+ * @return actual validated value
+ */
+ public static Object validate(String name, Object val, boolean isRuleVal) {
+ if (val == null) return null;
+ ValidateInfo info = validatetypes.get(name);
+ if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = validatetypes.get("STRING");
+ if (info == null) throw new RuntimeException("Unknown type :" + name);
+ if (info.type == Double.class) {
+ Double num = parseDouble(name, val);
+ if (isRuleVal) {
+ if (info.min != null)
+ if (Double.compare(num, (Double) info.min) == -1)
+ throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+ if (info.max != null)
+ if (Double.compare(num, (Double) info.max) == 1)
+ throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+ }
+ return num;
+ } else if (info.type == Long.class) {
+ Long num = parseLong(name, val);
+ if (isRuleVal) {
+ if (info.min != null)
+ if (num < info.min.longValue())
+ throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+ if (info.max != null)
+ if (num > info.max.longValue())
+ throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+ }
+ return num;
+ } else if (info.type == String.class) {
+ if (isRuleVal && info.vals != null && !info.vals.contains(val))
+ throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(info.vals, ','));
+ return val;
+ } else {
+ throw new RuntimeException("Invalid type ");
+ }
+ }
+
+ public static Long parseLong(String name, Object val) {
+ if (val == null) return null;
+ if (val instanceof Long) return (Long) val;
+ Number num = null;
+ if (val instanceof String) {
+ try {
+ num = Long.parseLong(((String) val).trim());
+ } catch (NumberFormatException e) {
+ try {
+ num = Double.parseDouble((String) val);
+ } catch (NumberFormatException e1) {
+ throw new RuntimeException(name + ": " + val + "not a valid number", e);
+ }
+ }
+
+ } else if (val instanceof Number) {
+ num = (Number) val;
+ }
+
+ if (num != null) {
+ return num.longValue();
+ }
+ throw new RuntimeException(name + ": " + val + "not a valid number");
+ }
+
+ public static Double parseDouble(String name, Object val) {
+ if (val == null) return null;
+ if (val instanceof Double) return (Double) val;
+ Number num = null;
+ if (val instanceof String) {
+ try {
+ num = Double.parseDouble((String) val);
+ } catch (NumberFormatException e) {
+ throw new RuntimeException(name + ": " + val + "not a valid number", e);
+ }
+
+ } else if (val instanceof Number) {
+ num = (Number) val;
+ }
+
+ if (num != null) {
+ return num.doubleValue();
+ }
+ throw new RuntimeException(name + ": " + val + "not a valid number");
+ }
+
+ private static final Map<String, ValidateInfo> validatetypes = new HashMap<>();
+
+ static {
+ validatetypes.put("collection", new ValidateInfo(String.class, null, null, null));
+ validatetypes.put("shard", new ValidateInfo(String.class, null, null, null));
+ validatetypes.put("replica", new ValidateInfo(Long.class, null, 0L, null));
+ validatetypes.put(ImplicitSnitch.PORT, new ValidateInfo(Long.class, null, 1L, 65535L));
+ validatetypes.put(ImplicitSnitch.DISK, new ValidateInfo(Double.class, null, 0d, Double.MAX_VALUE));
+ validatetypes.put(ImplicitSnitch.NODEROLE, new ValidateInfo(String.class, Collections.singleton("overseer"), null, null));
+ validatetypes.put(ImplicitSnitch.CORES, new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));
+ validatetypes.put(ImplicitSnitch.SYSLOADAVG, new ValidateInfo(Double.class, null, 0d, 100d));
+ validatetypes.put(ImplicitSnitch.HEAPUSAGE, new ValidateInfo(Double.class, null, 0d, null));
+ validatetypes.put("NUMBER", new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));//generic number validation
+ validatetypes.put("STRING", new ValidateInfo(String.class, null, null, null));//generic string validation
+ validatetypes.put("node", new ValidateInfo(String.class, null, null, null));
+ for (String ip : ImplicitSnitch.IP_SNITCHES) validatetypes.put(ip, new ValidateInfo(Long.class, null, 0L, 255L));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
new file mode 100644
index 0000000..e873625
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public interface ClusterDataProvider extends Closeable {
+ /**Get the value of each tag for a given node
+ *
+ * @param node node name
+ * @param tags tag names
+ * @return a map of tag vs value
+ */
+ Map<String, Object> getNodeValues(String node, Collection<String> tags);
+
+ /**
+ * Get the details of each replica in a node. It attempts to fetch as much details about
+ * the replica as mentioned in the keys list. It is not necessary to give al details
+ * <p>
+ * the format is {collection:shard :[{replicadetails}]}
+ */
+ Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
+
+ Collection<String> getNodes();
+
+ /**Get the collection-specific policy
+ */
+ String getPolicyNameByCollection(String coll);
+
+ @Override
+ default void close() throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
new file mode 100644
index 0000000..bf9c284
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.List;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.util.Pair;
+
+public class MoveReplicaSuggester extends Suggester {
+
+ @Override
+ SolrRequest init() {
+ SolrRequest operation = tryEachNode(true);
+ if (operation == null) operation = tryEachNode(false);
+ return operation;
+ }
+
+ SolrRequest tryEachNode(boolean strict) {
+ //iterate through elements and identify the least loaded
+ List<Clause.Violation> leastSeriousViolation = null;
+ Integer targetNodeIndex = null;
+ Integer fromNodeIndex = null;
+ ReplicaInfo fromReplicaInfo = null;
+ for (Pair<ReplicaInfo, Row> fromReplica : getValidReplicas(true, true, -1)) {
+ Row fromRow = fromReplica.second();
+ ReplicaInfo replicaInfo = fromReplica.first();
+ String coll = replicaInfo.collection;
+ String shard = replicaInfo.shard;
+ Pair<Row, ReplicaInfo> pair = fromRow.removeReplica(coll, shard);
+ Row tmpRow = pair.first();
+ if (tmpRow == null) {
+ //no such replica available
+ continue;
+ }
+ tmpRow.violations.clear();
+
+ final int i = getMatrix().indexOf(fromRow);
+ for (int j = getMatrix().size() - 1; j > i; j--) {
+ Row targetRow = getMatrix().get(j);
+ if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
+ targetRow = targetRow.addReplica(coll, shard);
+ targetRow.violations.clear();
+ List<Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getModifiedMatrix(getMatrix(), tmpRow, i), targetRow, j));
+ if (!containsNewErrors(errs) && isLessSerious(errs, leastSeriousViolation)) {
+ leastSeriousViolation = errs;
+ targetNodeIndex = j;
+ fromNodeIndex = i;
+ fromReplicaInfo = replicaInfo;
+ }
+ }
+ }
+ if (targetNodeIndex != null && fromNodeIndex != null) {
+ getMatrix().set(fromNodeIndex, getMatrix().get(fromNodeIndex).removeReplica(fromReplicaInfo.collection, fromReplicaInfo.shard).first());
+ getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(fromReplicaInfo.collection, fromReplicaInfo.shard));
+ return new CollectionAdminRequest.MoveReplica(
+ fromReplicaInfo.collection,
+ fromReplicaInfo.name,
+ getMatrix().get(targetNodeIndex).node);
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
new file mode 100644
index 0000000..e012718
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.FAIL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.NOT_APPLICABLE;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+
+import java.util.Objects;
+
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus;
+
+
+public enum Operand {
+ WILDCARD(ANY, Integer.MAX_VALUE) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return testVal == null ? NOT_APPLICABLE : PASS;
+ }
+
+ },
+ EQUAL("", 0) {
+ @Override
+ public int _delta(int expected, int actual) {
+ return expected - actual;
+ }
+ },
+ NOT_EQUAL("!", 2) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
+ }
+
+ @Override
+ public int _delta(int expected, int actual) {
+ return expected - actual;
+ }
+
+ },
+ GREATER_THAN(">", 1) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ if (testVal == null) return NOT_APPLICABLE;
+ if (ruleVal instanceof Double) {
+ return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == 1 ? PASS : FAIL;
+ }
+ return getLong(testVal) > getLong(ruleVal) ? PASS: FAIL ;
+ }
+
+ @Override
+ protected int _delta(int expected, int actual) {
+ return actual > expected ? 0 : (expected + 1) - actual;
+ }
+ },
+ LESS_THAN("<", 2) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ if (testVal == null) return NOT_APPLICABLE;
+ if (ruleVal instanceof Double) {
+ return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == -1 ? PASS : FAIL;
+ }
+ return getLong(testVal) < getLong(ruleVal) ? PASS: FAIL ;
+ }
+
+ @Override
+ protected int _delta(int expected, int actual) {
+ return actual < expected ? 0 : (expected ) - actual;
+ }
+
+ };
+ public final String operand;
+ final int priority;
+
+ Operand(String val, int priority) {
+ this.operand = val;
+ this.priority = priority;
+ }
+
+ public String toStr(Object expectedVal) {
+ return operand + expectedVal.toString();
+ }
+
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return Objects.equals(ruleVal, testVal) ? PASS : FAIL;
+ }
+
+ Long getLong(Object o) {
+ if (o instanceof Long) return (Long) o;
+ if(o instanceof Number ) return ((Number) o).longValue();
+ return Long.parseLong(String.valueOf(o));
+
+ }
+
+ public Integer delta(Object expected, Object actual) {
+ try {
+ Integer expectedInt = Integer.parseInt(String.valueOf(expected));
+ Integer actualInt = Integer.parseInt(String.valueOf(actual));
+ return _delta(expectedInt, actualInt);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ protected int _delta(int expected, int actual) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
new file mode 100644
index 0000000..ccb0dee
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.stream.Collectors.toList;
+
+/*The class that reads, parses and applies policies specified in
+ * autoscaling.json
+ *
+ * Create one instance of this class per unique autoscaling.json.
+ * This is immutable and is thread-safe
+ *
+ * Create a fresh new session for each use
+ *
+ */
+public class Policy implements MapWriter {
+ public static final String POLICY = "policy";
+ public static final String EACH = "#EACH";
+ public static final String ANY = "#ANY";
+ public static final String CLUSTER_POLICY = "cluster-policy";
+ public static final String CLUSTER_PREFERENCE = "cluster-preferences";
+ public static final Set<String> GLOBAL_ONLY_TAGS = Collections.singleton("cores");
+ final Map<String, List<Clause>> policies = new HashMap<>();
+ final List<Clause> clusterPolicy;
+ final List<Preference> clusterPreferences;
+ final List<String> params;
+
+
+ public Policy(Map<String, Object> jsonMap) {
+
+ clusterPreferences = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_PREFERENCE, emptyList())).stream()
+ .map(Preference::new)
+ .collect(toList());
+ for (int i = 0; i < clusterPreferences.size() - 1; i++) {
+ Preference preference = clusterPreferences.get(i);
+ preference.next = clusterPreferences.get(i + 1);
+ }
+ if (clusterPreferences.isEmpty()) {
+ clusterPreferences.add(new Preference((Map<String, Object>) Utils.fromJSONString("{minimize : cores, precision:1}")));
+ }
+ SortedSet<String> paramsOfInterest = new TreeSet<>();
+ for (Preference preference : clusterPreferences) {
+ if (paramsOfInterest.contains(preference.name.name())) {
+ throw new RuntimeException(preference.name + " is repeated");
+ }
+ paramsOfInterest.add(preference.name.toString());
+ }
+ this.params = new ArrayList<>(paramsOfInterest);
+
+ clusterPolicy = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_POLICY, emptyList())).stream()
+ .map(Clause::new)
+ .filter(clause -> {
+ clause.addTags(params);
+ return true;
+ })
+ .collect(Collectors.toList());
+
+ ((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault("policies", emptyMap())).forEach((s, l1) ->
+ this.policies.put(s, l1.stream()
+ .map(Clause::new)
+ .filter(clause -> {
+ if (!clause.isPerCollectiontag())
+ throw new RuntimeException(clause.globalTag.name + " is only allowed in 'cluster-policy'");
+ clause.addTags(params);
+ return true;
+ })
+ .sorted()
+ .collect(toList())));
+ }
+
+ public List<Clause> getClusterPolicy() {
+ return clusterPolicy;
+ }
+
+ public List<Preference> getClusterPreferences() {
+ return clusterPreferences;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ if (!policies.isEmpty()) {
+ ew.put("policies", (MapWriter) ew1 -> {
+ for (Map.Entry<String, List<Clause>> e : policies.entrySet()) {
+ ew1.put(e.getKey(), e.getValue());
+ }
+ });
+ }
+ if (!clusterPreferences.isEmpty()) {
+ ew.put("preferences", (IteratorWriter) iw -> {
+ for (Preference p : clusterPreferences) iw.add(p);
+ });
+ }
+
+ }
+
+ /*This stores the logical state of the system, given a policy and
+ * a cluster state.
+ *
+ */
+ public class Session implements MapWriter {
+ final List<String> nodes;
+ final ClusterDataProvider dataProvider;
+ final List<Row> matrix;
+ Set<String> collections = new HashSet<>();
+ List<Clause> expandedClauses;
+ List<Violation> violations = new ArrayList<>();
+
+ private Session(List<String> nodes, ClusterDataProvider dataProvider,
+ List<Row> matrix, List<Clause> expandedClauses) {
+ this.nodes = nodes;
+ this.dataProvider = dataProvider;
+ this.matrix = matrix;
+ this.expandedClauses = expandedClauses;
+ }
+
+ Session(ClusterDataProvider dataProvider) {
+ this.nodes = new ArrayList<>(dataProvider.getNodes());
+ this.dataProvider = dataProvider;
+ for (String node : nodes) {
+ collections.addAll(dataProvider.getReplicaInfo(node, Collections.emptyList()).keySet());
+ }
+
+ expandedClauses = clusterPolicy.stream()
+ .filter(clause -> !clause.isPerCollectiontag())
+ .collect(Collectors.toList());
+
+ for (String c : collections) {
+ addClausesForCollection(dataProvider, c);
+ }
+
+ Collections.sort(expandedClauses);
+
+ matrix = new ArrayList<>(nodes.size());
+ for (String node : nodes) matrix.add(new Row(node, params, dataProvider));
+ applyRules();
+ }
+
+ private void addClausesForCollection(ClusterDataProvider dataProvider, String c) {
+ String p = dataProvider.getPolicyNameByCollection(c);
+ if (p != null) {
+ List<Clause> perCollPolicy = policies.get(p);
+ if (perCollPolicy == null)
+ throw new RuntimeException(StrUtils.formatString("Policy for collection {0} is {1} . It does not exist", c, p));
+ }
+ expandedClauses.addAll(mergePolicies(c, policies.getOrDefault(p, emptyList()), clusterPolicy));
+ }
+
+ Session copy() {
+ return new Session(nodes, dataProvider, getMatrixCopy(), expandedClauses);
+ }
+
+ List<Row> getMatrixCopy() {
+ return matrix.stream()
+ .map(Row::copy)
+ .collect(Collectors.toList());
+ }
+
+ Policy getPolicy() {
+ return Policy.this;
+
+ }
+
+ /**
+ * Apply the preferences and conditions
+ */
+ private void applyRules() {
+ if (!clusterPreferences.isEmpty()) {
+ //this is to set the approximate value according to the precision
+ ArrayList<Row> tmpMatrix = new ArrayList<>(matrix);
+ for (Preference p : clusterPreferences) {
+ Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false));
+ p.setApproxVal(tmpMatrix);
+ }
+ //approximate values are set now. Let's do recursive sorting
+ Collections.sort(matrix, (Row r1, Row r2) -> {
+ int result = clusterPreferences.get(0).compare(r1, r2, true);
+ if (result == 0) result = clusterPreferences.get(0).compare(r1, r2, false);
+ return result;
+ });
+ }
+
+ for (Clause clause : expandedClauses) {
+ List<Violation> errs = clause.test(matrix);
+ violations.addAll(errs);
+ }
+ }
+
+ public List<Violation> getViolations() {
+ return violations;
+ }
+
+ public Suggester getSuggester(CollectionAction action) {
+ Suggester op = ops.get(action).get();
+ if (op == null) throw new UnsupportedOperationException(action.toString() + "is not supported");
+ op._init(this);
+ return op;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (int i = 0; i < matrix.size(); i++) {
+ Row row = matrix.get(i);
+ ew.put(row.node, row);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(toMap(new LinkedHashMap<>()));
+ }
+
+ public List<Row> getSorted() {
+ return Collections.unmodifiableList(matrix);
+ }
+ }
+
+
+ public Session createSession(ClusterDataProvider dataProvider) {
+ return new Session(dataProvider);
+ }
+
+ public enum SortParam {
+ freedisk(0, Integer.MAX_VALUE), cores(0, Integer.MAX_VALUE), heapUsage(0, Integer.MAX_VALUE), sysLoadAvg(0, 100);
+
+ public final int min,max;
+
+ SortParam(int min, int max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ static SortParam get(String m) {
+ for (SortParam p : values()) if (p.name().equals(m)) return p;
+ throw new RuntimeException(StrUtils.formatString("Invalid sort {0} Sort must be on one of these {1}", m, Arrays.asList(values())));
+ }
+ }
+
+ enum Sort {
+ maximize(1), minimize(-1);
+ final int sortval;
+
+ Sort(int i) {
+ sortval = i;
+ }
+
+ static Sort get(Map<String, Object> m) {
+ if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) {
+ throw new RuntimeException("Cannot have both 'maximize' and 'minimize'");
+ }
+ if (m.containsKey(maximize.name())) return maximize;
+ if (m.containsKey(minimize.name())) return minimize;
+ throw new RuntimeException("must have either 'maximize' or 'minimize'");
+ }
+ }
+
+
+ public static class ReplicaInfo implements MapWriter {
+ final String name;
+ String core, collection, shard;
+ Map<String, Object> variables;
+
+ public ReplicaInfo(String name, String coll, String shard, Map<String, Object> vals) {
+ this.name = name;
+ this.variables = vals;
+ this.collection = coll;
+ this.shard = shard;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, variables);
+ }
+
+ public String getCore() {
+ return core;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public String getShard() {
+ return shard;
+ }
+ }
+
+
+ /* A suggester is capable of suggesting a collection operation
+ * given a particular session. Before it suggests a new operation,
+ * it ensures that ,
+ * a) load is reduced on the most loaded node
+ * b) it causes no new violations
+ *
+ */
+ public static abstract class Suggester {
+ protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
+ Policy.Session session;
+ SolrRequest operation;
+ protected List<Violation> originalViolations = new ArrayList<>();
+ private boolean isInitialized = false;
+
+ private void _init(Session session) {
+ this.session = session.copy();
+ }
+
+ public Suggester hint(Hint hint, Object value) {
+ if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+ ((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).add(value);
+ } else {
+ hints.put(hint, value);
+ }
+ return this;
+ }
+
+ abstract SolrRequest init();
+
+
+ public SolrRequest getOperation() {
+ if (!isInitialized) {
+ String coll = (String) hints.get(Hint.COLL);
+ String shard = (String) hints.get(Hint.SHARD);
+ // if this is not a known collection from the existing clusterstate,
+ // then add it
+ if (session.matrix.stream().noneMatch(row -> row.collectionVsShardVsReplicas.containsKey(coll))) {
+ session.addClausesForCollection(session.dataProvider, coll);
+ Collections.sort(session.expandedClauses);
+ }
+ if (coll != null) {
+ for (Row row : session.matrix) {
+ if (!row.collectionVsShardVsReplicas.containsKey(coll)) row.collectionVsShardVsReplicas.put(coll, new HashMap<>());
+ if (shard != null) {
+ Map<String, List<ReplicaInfo>> shardInfo = row.collectionVsShardVsReplicas.get(coll);
+ if (!shardInfo.containsKey(shard)) shardInfo.put(shard, new ArrayList<>());
+ }
+ }
+ }
+ session.applyRules();
+ originalViolations.addAll(session.getViolations());
+ this.operation = init();
+ isInitialized = true;
+ }
+ return operation;
+ }
+
+ public Session getSession() {
+ return session;
+ }
+
+ List<Row> getMatrix() {
+ return session.matrix;
+
+ }
+
+ //check if the fresh set of violations is less serious than the last set of violations
+ boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
+ if (old == null || fresh.size() < old.size()) return true;
+ if (fresh.size() == old.size()) {
+ for (int i = 0; i < fresh.size(); i++) {
+ Violation freshViolation = fresh.get(i);
+ Violation oldViolation = null;
+ for (Violation v : old) {
+ if (v.equals(freshViolation)) oldViolation = v;
+ }
+ if (oldViolation != null && freshViolation.isLessSerious(oldViolation)) return true;
+ }
+ }
+ return false;
+ }
+
+ boolean containsNewErrors(List<Violation> violations) {
+ for (Violation v : violations) {
+ int idx = originalViolations.indexOf(v);
+ if (idx < 0 || originalViolations.get(idx).isLessSerious(v)) return true;
+ }
+ return false;
+ }
+
+ List<Pair<ReplicaInfo, Row>> getValidReplicas(boolean sortDesc, boolean isSource, int until) {
+ List<Pair<Policy.ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
+
+ if (sortDesc) {
+ if (until == -1) until = getMatrix().size();
+ for (int i = 0; i < until; i++) addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+ } else {
+ if (until == -1) until = 0;
+ for (int i = getMatrix().size() - 1; i >= until; i--)
+ addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+ }
+ return allPossibleReplicas;
+ }
+
+ void addReplicaToList(Row r, boolean isSource, List<Pair<Policy.ReplicaInfo, Row>> replicaList) {
+ if (!isAllowed(r.node, isSource ? Hint.SRC_NODE : Hint.TARGET_NODE)) return;
+ for (Map.Entry<String, Map<String, List<Policy.ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
+ if (!isAllowed(e.getKey(), Hint.COLL)) continue;
+ for (Map.Entry<String, List<Policy.ReplicaInfo>> shard : e.getValue().entrySet()) {
+ if (!isAllowed(e.getKey(), Hint.SHARD)) continue;
+ replicaList.add(new Pair<>(shard.getValue().get(0), r));
+ }
+ }
+ }
+
+ protected List<Violation> testChangedMatrix(boolean strict, List<Row> rows) {
+ List<Violation> errors = new ArrayList<>();
+ for (Clause clause : session.expandedClauses) {
+ if (strict || clause.strict) {
+ List<Violation> errs = clause.test(rows);
+ if (!errs.isEmpty()) {
+ errors.addAll(errs);
+ }
+ }
+ }
+ return errors;
+ }
+
+ ArrayList<Row> getModifiedMatrix(List<Row> matrix, Row tmpRow, int i) {
+ ArrayList<Row> copy = new ArrayList<>(matrix);
+ copy.set(i, tmpRow);
+ return copy;
+ }
+
+ protected boolean isAllowed(Object v, Hint hint) {
+ Object hintVal = hints.get(hint);
+ if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+ Set set = (Set) hintVal;
+ return set == null || set.contains(v);
+ } else {
+ return hintVal == null || Objects.equals(v, hintVal);
+ }
+ }
+
+ public enum Hint {
+ COLL, SHARD, SRC_NODE, TARGET_NODE
+ }
+
+
+ }
+
+ public static List<Clause> mergePolicies(String coll,
+ List<Clause> collPolicy,
+ List<Clause> globalPolicy) {
+
+ List<Clause> merged = insertColl(coll, collPolicy);
+ List<Clause> global = insertColl(coll, globalPolicy);
+ merged.addAll(global.stream()
+ .filter(clusterPolicyClause -> merged.stream().noneMatch(perCollPolicy -> perCollPolicy.doesOverride(clusterPolicyClause)))
+ .collect(Collectors.toList()));
+ return merged;
+ }
+
+ /**
+ * Insert the collection name into the clauses where collection is not specified
+ */
+ static List<Clause> insertColl(String coll, Collection<Clause> conditions) {
+ return conditions.stream()
+ .filter(Clause::isPerCollectiontag)
+ .map(clause -> {
+ Map<String, Object> copy = new LinkedHashMap<>(clause.original);
+ if (!copy.containsKey("collection")) copy.put("collection", coll);
+ return new Clause(copy);
+ })
+ .filter(it -> (it.collection.isPass(coll)))
+ .collect(Collectors.toList());
+
+ }
+
+ private static final Map<CollectionAction, Supplier<Suggester>> ops = new HashMap<>();
+
+ static {
+ ops.put(CollectionAction.ADDREPLICA, () -> new AddReplicaSuggester());
+ ops.put(CollectionAction.MOVEREPLICA, () -> new MoveReplicaSuggester());
+ }
+
+ public Map<String, List<Clause>> getPolicies() {
+ return policies;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
new file mode 100644
index 0000000..deea175
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.Utils;
+
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+
+public class PolicyHelper {
+ public static Map<String, List<String>> getReplicaLocations(String collName, Map<String, Object> autoScalingJson,
+ ClusterDataProvider cdp,
+ Map<String, String> optionalPolicyMapping,
+ List<String> shardNames,
+ int repFactor,
+ List<String> nodesList) {
+ Map<String, List<String>> positionMapping = new HashMap<>();
+ for (String shardName : shardNames) positionMapping.put(shardName, new ArrayList<>(repFactor));
+ if (optionalPolicyMapping != null) {
+ final ClusterDataProvider delegate = cdp;
+ cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return delegate.getNodeValues(node, tags);
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return delegate.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return delegate.getNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return optionalPolicyMapping.containsKey(coll) ?
+ optionalPolicyMapping.get(coll) :
+ delegate.getPolicyNameByCollection(coll);
+ }
+ };
+
+ }
+
+
+ Policy policy = new Policy(autoScalingJson);
+ Policy.Session session = policy.createSession(cdp);
+ for (String shardName : shardNames) {
+ for (int i = 0; i < repFactor; i++) {
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, collName)
+ .hint(Hint.SHARD, shardName);
+ if (nodesList != null) {
+ for (String nodeName : nodesList) {
+ suggester = suggester.hint(Hint.TARGET_NODE, nodeName);
+ }
+ }
+ SolrRequest op = suggester.getOperation();
+ if (op == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(Utils.getDeepCopy(session.expandedClauses, 4, true)));
+ }
+ session = suggester.getSession();
+ positionMapping.get(shardName).add(op.getParams().get(CoreAdminParams.NODE));
+ }
+ }
+
+ return positionMapping;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
new file mode 100644
index 0000000..bb45628
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+public class Preference implements MapWriter {
+ final Policy.SortParam name;
+ Integer precision;
+ final Policy.Sort sort;
+ Preference next;
+ public int idx;
+ private final Map original;
+
+ Preference(Map<String, Object> m) {
+ this.original = Utils.getDeepCopy(m,3);
+ sort = Policy.Sort.get(m);
+ name = Policy.SortParam.get(m.get(sort.name()).toString());
+ Object p = m.getOrDefault("precision", 0);
+ precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
+ if (precision < 0) {
+ throw new RuntimeException("precision must be a positive value ");
+ }
+ if(precision< name.min || precision> name.max){
+ throw new RuntimeException(StrUtils.formatString("invalid precision value {0} must lie between {1} and {1}",
+ precision, name.min, name.max ) );
+ }
+
+ }
+
+ // there are 2 modes of compare.
+ // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
+ // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
+ int compare(Row r1, Row r2, boolean useApprox) {
+ Object o1 = useApprox ? r1.cells[idx].approxVal : r1.cells[idx].val;
+ Object o2 = useApprox ? r2.cells[idx].approxVal : r2.cells[idx].val;
+ int result = 0;
+ if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
+ else if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
+ else if (!o1.getClass().getName().equals(o2.getClass().getName())) {
+ throw new RuntimeException("Unable to compare " + o1 + " of type: " + o1.getClass().getName() + " from " + r1.cells[idx].toString() + " and " + o2 + " of type: " + o2.getClass().getName() + " from " + r2.cells[idx].toString());
+ }
+ return result == 0 ? (next == null ? 0 : next.compare(r1, r2, useApprox)) : sort.sortval * result;
+ }
+
+ //sets the new value according to precision in val_
+ void setApproxVal(List<Row> tmpMatrix) {
+ Object prevVal = null;
+ for (Row row : tmpMatrix) {
+ prevVal = row.cells[idx].approxVal =
+ (prevVal == null || Double.compare(Math.abs(((Number) prevVal).doubleValue() - ((Number) row.cells[idx].val).doubleValue()), precision) > 0) ?
+ row.cells[idx].val :
+ prevVal;
+ }
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (Object o : original.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ ew.put(String.valueOf(e.getKey()), e.getValue());
+ }
+ }
+
+ public Policy.SortParam getName() {
+ return name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
new file mode 100644
index 0000000..76c8c57
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.Utils;
+
+import static org.apache.solr.common.params.CoreAdminParams.NODE;
+
+
+public class Row implements MapWriter {
+ public final String node;
+ final Cell[] cells;
+ public Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas;
+ List<Clause> violations = new ArrayList<>();
+ boolean anyValueMissing = false;
+
+ public Row(String node, List<String> params, ClusterDataProvider dataProvider) {
+ collectionVsShardVsReplicas = dataProvider.getReplicaInfo(node, params);
+ if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>();
+ this.node = node;
+ cells = new Cell[params.size()];
+ Map<String, Object> vals = dataProvider.getNodeValues(node, params);
+ for (int i = 0; i < params.size(); i++) {
+ String s = params.get(i);
+ cells[i] = new Cell(i, s, Clause.validate(s,vals.get(s), false));
+ if (NODE.equals(s)) cells[i].val = node;
+ if (cells[i].val == null) anyValueMissing = true;
+ }
+ }
+
+ public Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, List<Clause> violations) {
+ this.node = node;
+ this.cells = new Cell[cells.length];
+ for (int i = 0; i < this.cells.length; i++) {
+ this.cells[i] = cells[i].copy();
+
+ }
+ this.anyValueMissing = anyValueMissing;
+ this.collectionVsShardVsReplicas = collectionVsShardVsReplicas;
+ this.violations = violations;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(node, (IteratorWriter) iw -> {
+ iw.add((MapWriter) e -> e.put("replicas", collectionVsShardVsReplicas));
+ for (Cell cell : cells) iw.add(cell);
+ });
+ }
+
+ Row copy() {
+ return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(collectionVsShardVsReplicas, 3), new ArrayList<>(violations));
+ }
+
+ Object getVal(String name) {
+ for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return node;
+ }
+
+ // this adds a replica to the replica info
+ public Row addReplica(String coll, String shard) {
+ Row row = copy();
+ Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
+ List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
+ replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, new HashMap<>()));
+ for (Cell cell : row.cells) {
+ if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() + 1;
+ }
+ return row;
+
+ }
+
+ public Pair<Row, ReplicaInfo> removeReplica(String coll, String shard) {
+ Row row = copy();
+ Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.get(coll);
+ if (c == null) return null;
+ List<ReplicaInfo> s = c.get(shard);
+ if (s == null || s.isEmpty()) return null;
+ for (Cell cell : row.cells) {
+ if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() -1;
+ }
+ return new Pair(row, s.remove(0));
+
+ }
+
+ public Cell[] getCells() {
+ return cells;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java
new file mode 100644
index 0000000..620f57d
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common classes for autoscaling parsing filtering nodes and sorting
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
index e40f32b..5b61822 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
@@ -32,10 +32,10 @@ import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
-import org.apache.solr.cloud.autoscaling.ClusterDataProvider;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 740de34..f039ba4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -46,7 +46,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
deleted file mode 100644
index 354851e..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.util.List;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.autoscaling.Policy.Suggester;
-
-class AddReplicaSuggester extends Suggester {
-
- SolrRequest init() {
- SolrRequest operation = tryEachNode(true);
- if (operation == null) operation = tryEachNode(false);
- return operation;
- }
-
- SolrRequest tryEachNode(boolean strict) {
- String coll = (String) hints.get(Hint.COLL);
- String shard = (String) hints.get(Hint.SHARD);
- if (coll == null || shard == null)
- throw new RuntimeException("add-replica requires 'collection' and 'shard'");
- //iterate through elements and identify the least loaded
-
- List<Clause.Violation> leastSeriousViolation = null;
- Integer targetNodeIndex = null;
- for (int i = getMatrix().size() - 1; i >= 0; i--) {
- Row row = getMatrix().get(i);
- if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
- Row tmpRow = row.addReplica(coll, shard);
- tmpRow.violations.clear();
-
- List<Clause.Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
- if(!containsNewErrors(errs)) {
- if(isLessSerious(errs, leastSeriousViolation)){
- leastSeriousViolation = errs;
- targetNodeIndex = i;
- }
- }
- }
-
- if (targetNodeIndex != null) {// there are no rule violations
- getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
- return CollectionAdminRequest
- .addReplicaToShard(coll, shard)
- .setNode(getMatrix().get(targetNodeIndex).node);
- }
-
- return null;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
deleted file mode 100644
index 0f2b24b..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.util.Utils;
-
-class Cell implements MapWriter {
- final int index;
- final String name;
- Object val, approxVal;
-
- Cell(int index, String name, Object val) {
- this.index = index;
- this.name = name;
- this.val = val;
- }
-
- Cell(int index, String name, Object val, Object approxVal) {
- this.index = index;
- this.name = name;
- this.val = val;
- this.approxVal = approxVal;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(name, val);
- }
-
- @Override
- public String toString() {
- return Utils.toJSONString(this.toMap(new HashMap<>()));
- }
-
- public Cell copy() {
- return new Cell(index, name, val, approxVal);
- }
-}
[05/18] lucene-solr:feature/autoscaling: Fix three ex[c]eption typos.
Posted by sh...@apache.org.
Fix three ex[c]eption typos.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/808171ac
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/808171ac
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/808171ac
Branch: refs/heads/feature/autoscaling
Commit: 808171ac0001e952109c4a492b0a0121c60cea75
Parents: 701c73d
Author: Christine Poerschke <cp...@apache.org>
Authored: Tue Jun 27 14:46:39 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Tue Jun 27 15:07:40 2017 +0100
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java | 4 ++--
solr/core/src/java/org/apache/solr/core/CoreContainer.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/808171ac/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java b/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
index 3a5d41c..6bdc943 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
@@ -235,7 +235,7 @@ public abstract class Analyzer implements Closeable {
}
filteredText = builder.toString();
} catch (IOException e) {
- throw new IllegalStateException("Normalization threw an unexpected exeption", e);
+ throw new IllegalStateException("Normalization threw an unexpected exception", e);
}
final AttributeFactory attributeFactory = attributeFactory(fieldName);
@@ -258,7 +258,7 @@ public abstract class Analyzer implements Closeable {
return term;
}
} catch (IOException e) {
- throw new IllegalStateException("Normalization threw an unexpected exeption", e);
+ throw new IllegalStateException("Normalization threw an unexpected exception", e);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/808171ac/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 6055acf..90bbf5d 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1032,7 +1032,7 @@ public class CoreContainer {
}
}
- // If no CorruptIndexExeption, nothing we can try here
+ // If no CorruptIndexException, nothing we can try here
if (cause == null) throw original;
CoreInitFailedAction action = CoreInitFailedAction.valueOf(System.getProperty(CoreInitFailedAction.class.getSimpleName(), "none"));
[15/18] lucene-solr:feature/autoscaling: Merge branch 'master' into
feature/autoscaling
Posted by sh...@apache.org.
Merge branch 'master' into feature/autoscaling
# Conflicts:
# solr/CHANGES.txt
# solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
# solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
# solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
# solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
# solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
# solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
# solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
# solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
# solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4239896e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4239896e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4239896e
Branch: refs/heads/feature/autoscaling
Commit: 4239896e6dfeb9ef1b41313e025b5e2f90675817
Parents: 257ba05 13a3ae2
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Jun 28 11:09:19 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Jun 28 11:09:19 2017 +0530
----------------------------------------------------------------------
.../lucene/spatial-extras/spatial-extras.iml | 5 +-
lucene/CHANGES.txt | 8 +
.../byTask/feeds/SpatialFileQueryMaker.java | 15 +-
.../org/apache/lucene/analysis/Analyzer.java | 4 +-
.../org/apache/lucene/search/DoubleValues.java | 21 +
.../lucene/search/DoubleValuesSource.java | 216 ++++----
.../apache/lucene/search/LongValuesSource.java | 82 ++-
.../lucene/search/TestDoubleValuesSource.java | 16 +-
.../facet/range/TestRangeFacetCounts.java | 75 +--
.../queries/function/FunctionScoreQuery.java | 9 -
.../lucene/queries/function/ValueSource.java | 260 +++++++---
.../function/TestFunctionScoreExplanations.java | 8 +-
.../function/TestFunctionScoreQuery.java | 91 +++-
lucene/spatial-extras/build.xml | 8 +-
.../org/apache/lucene/spatial/ShapeValues.java | 41 ++
.../lucene/spatial/ShapeValuesSource.java | 34 ++
.../apache/lucene/spatial/SpatialStrategy.java | 19 +-
.../bbox/BBoxOverlapRatioValueSource.java | 7 +-
.../spatial/bbox/BBoxSimilarityValueSource.java | 79 ++-
.../lucene/spatial/bbox/BBoxStrategy.java | 13 +-
.../lucene/spatial/bbox/BBoxValueSource.java | 74 +--
.../composite/CompositeSpatialStrategy.java | 14 +-
.../spatial/composite/CompositeVerifyQuery.java | 32 +-
.../composite/IntersectsRPTVerifyQuery.java | 21 +-
.../prefix/NumberRangePrefixTreeStrategy.java | 4 +-
.../spatial/prefix/PrefixTreeStrategy.java | 4 +-
.../serialized/SerializedDVStrategy.java | 110 +---
.../spatial/util/CachingDoubleValueSource.java | 61 ++-
.../util/DistanceToShapeValueSource.java | 68 +--
.../util/ReciprocalDoubleValuesSource.java | 96 ++++
.../spatial/util/ShapeAreaValueSource.java | 67 +--
.../ShapeFieldCacheDistanceValueSource.java | 59 ++-
.../spatial/util/ShapePredicateValueSource.java | 113 -----
.../spatial/util/ShapeValuesPredicate.java | 99 ++++
.../spatial/vector/DistanceValueSource.java | 72 +--
.../spatial/vector/PointVectorStrategy.java | 95 +++-
.../lucene/spatial/DistanceStrategyTest.java | 6 -
.../apache/lucene/spatial/SpatialExample.java | 12 +-
.../apache/lucene/spatial/StrategyTestCase.java | 39 +-
.../lucene/spatial/spatial4j/Geo3dRptTest.java | 2 +-
.../Geo3dShapeRectRelationTestCase.java | 2 +-
.../DocumentValueSourceDictionaryTest.java | 15 +
solr/CHANGES.txt | 10 +-
solr/bin/solr | 13 +-
.../ltr/model/MultipleAdditiveTreesModel.java | 16 +-
.../src/java/org/apache/solr/cloud/Assign.java | 6 +-
.../cloud/OverseerCollectionMessageHandler.java | 5 +-
.../org/apache/solr/cloud/ZkController.java | 19 +-
.../cloud/autoscaling/AutoScalingHandler.java | 13 +-
.../org/apache/solr/core/CoreContainer.java | 2 +-
.../solr/handler/admin/CollectionsHandler.java | 2 +-
.../org/apache/solr/legacy/BBoxStrategy.java | 13 +-
.../org/apache/solr/legacy/BBoxValueSource.java | 82 +--
.../apache/solr/legacy/DistanceValueSource.java | 81 ++-
.../apache/solr/legacy/PointVectorStrategy.java | 17 +-
.../transform/GeoTransformerFactory.java | 33 +-
.../solr/schema/AbstractSpatialFieldType.java | 10 +-
.../java/org/apache/solr/schema/BBoxField.java | 8 +-
.../solr/schema/LatLonPointSpatialField.java | 46 +-
.../schema/RptWithGeometrySpatialField.java | 60 +--
.../apache/solr/schema/ZkIndexSchemaReader.java | 105 ++--
.../distance/GeoDistValueSourceParser.java | 8 +-
.../configsets/_default/conf/solrconfig.xml | 2 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 2 +-
.../apache/solr/schema/SchemaWatcherTest.java | 56 +++
.../apache/solr/search/TestSolr4Spatial.java | 10 +-
solr/solr-ref-guide/src/graph-traversal.adoc | 296 +++++------
solr/solr-ref-guide/src/stream-sources.adoc | 4 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 73 +++
.../client/solrj/cloud/autoscaling/Cell.java | 69 +++
.../client/solrj/cloud/autoscaling/Clause.java | 480 ++++++++++++++++++
.../cloud/autoscaling/ClusterDataProvider.java | 52 ++
.../cloud/autoscaling/MoveReplicaSuggester.java | 83 +++
.../client/solrj/cloud/autoscaling/Operand.java | 124 +++++
.../client/solrj/cloud/autoscaling/Policy.java | 499 +++++++++++++++++++
.../solrj/cloud/autoscaling/PolicyHelper.java | 96 ++++
.../solrj/cloud/autoscaling/Preference.java | 92 ++++
.../client/solrj/cloud/autoscaling/Row.java | 140 ++++++
.../solrj/cloud/autoscaling/package-info.java | 23 +
.../solrj/impl/SolrClientDataProvider.java | 4 +-
.../solrj/request/CollectionAdminRequest.java | 2 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 73 ---
.../org/apache/solr/cloud/autoscaling/Cell.java | 57 ---
.../apache/solr/cloud/autoscaling/Clause.java | 467 -----------------
.../cloud/autoscaling/ClusterDataProvider.java | 52 --
.../cloud/autoscaling/MoveReplicaSuggester.java | 83 ---
.../apache/solr/cloud/autoscaling/Operand.java | 124 -----
.../apache/solr/cloud/autoscaling/Policy.java | 497 ------------------
.../solr/cloud/autoscaling/PolicyHelper.java | 96 ----
.../solr/cloud/autoscaling/Preference.java | 88 ----
.../org/apache/solr/cloud/autoscaling/Row.java | 135 -----
.../solr/cloud/autoscaling/package-info.java | 23 -
.../apache/solr/common/cloud/DocCollection.java | 2 +-
.../solr/cloud/autoscaling/TestPolicy.java | 20 +-
94 files changed, 3426 insertions(+), 3023 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/CHANGES.txt
----------------------------------------------------------------------
diff --cc solr/CHANGES.txt
index d614519,3f25c26..d8c90a2
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@@ -224,13 -211,9 +224,16 @@@ Bug Fixe
* SOLR-10948: Fix extraction component to treat DatePointField the same as TrieDateField (hossman)
+ * SOLR-10506: Fix memory leak (upon collection reload or ZooKeeper session expiry) in ZkIndexSchemaReader.
+ (Torsten Bøgh Köster, Christine Poerschke, Jörg Rathlev, Mike Drob)
+
+* SOLR-10602: Triggers should be able to restore state from old instances when taking over. (shalin)
+
+* SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling
+ config watcher is fired. (shalin)
+
+* SOLR-10738: TriggerAction is initialised even if the trigger is never scheduled. (shalin)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 5e16b5d,c6051d7..3ee1fde
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@@ -42,9 -43,6 +43,9 @@@ import org.apache.solr.client.solrj.imp
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.cloud.autoscaling.AutoScaling;
+import org.apache.solr.cloud.autoscaling.AutoScalingHandler;
- import org.apache.solr.cloud.autoscaling.Policy;
++import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.ReplicaAssigner.Position;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
index 0000000,01149f3..7b00c87
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@@ -1,0 -1,69 +1,73 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import java.util.List;
+
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
++import org.apache.solr.common.cloud.Replica;
+
+ class AddReplicaSuggester extends Suggester {
+
+ SolrRequest init() {
+ SolrRequest operation = tryEachNode(true);
+ if (operation == null) operation = tryEachNode(false);
+ return operation;
+ }
+
+ SolrRequest tryEachNode(boolean strict) {
+ String coll = (String) hints.get(Hint.COLL);
+ String shard = (String) hints.get(Hint.SHARD);
++ Replica.Type type = Replica.Type.get((String) hints.get(Hint.REPLICATYPE));
+ if (coll == null || shard == null)
+ throw new RuntimeException("add-replica requires 'collection' and 'shard'");
+ //iterate through elements and identify the least loaded
+
+ List<Clause.Violation> leastSeriousViolation = null;
+ Integer targetNodeIndex = null;
+ for (int i = getMatrix().size() - 1; i >= 0; i--) {
+ Row row = getMatrix().get(i);
++ if(!row.isLive) continue;
+ if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
- Row tmpRow = row.addReplica(coll, shard);
++ Row tmpRow = row.addReplica(coll, shard, type);
+ tmpRow.violations.clear();
+
+ List<Clause.Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
+ if(!containsNewErrors(errs)) {
+ if(isLessSerious(errs, leastSeriousViolation)){
+ leastSeriousViolation = errs;
+ targetNodeIndex = i;
+ }
+ }
+ }
+
+ if (targetNodeIndex != null) {// there are no rule violations
- getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
++ getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard, type));
+ return CollectionAdminRequest
+ .addReplicaToShard(coll, shard)
++ .setType(type)
+ .setNode(getMatrix().get(targetNodeIndex).node);
+ }
+
+ return null;
+ }
+
+
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index 0000000,ff56627..8a5a121
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@@ -1,0 -1,472 +1,480 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.HashSet;
++import java.util.LinkedHashMap;
+ import java.util.List;
++import java.util.Locale;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.Optional;
+ import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
+
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+ import org.apache.solr.common.MapWriter;
++import org.apache.solr.common.cloud.Replica;
+ import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+ import org.apache.solr.common.util.StrUtils;
+ import org.apache.solr.common.util.Utils;
+
+ import static java.util.Collections.singletonMap;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+ import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
+ import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
+ import static org.apache.solr.common.params.CoreAdminParams.SHARD;
+
+ // a set of conditions in a policy
+ public class Clause implements MapWriter, Comparable<Clause> {
+ public Map<String, Object> original;
+ public Condition collection, shard, replica, tag, globalTag;
++ public final Replica.Type type;
+
+ boolean strict = true;
+
+ public Clause(Map<String, Object> m) {
+ this.original = m;
++ String type = (String) m.get("type");
++ this.type = type == null || ANY.equals(type) ? null : Replica.Type.valueOf(type.toUpperCase(Locale.ROOT));
+ strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true")));
+ Optional<String> globalTagName = m.keySet().stream().filter(Policy.GLOBAL_ONLY_TAGS::contains).findFirst();
+ if (globalTagName.isPresent()) {
+ globalTag = parse(globalTagName.get(), m);
+ if (m.size() > 2) {
+ throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + Utils.toJSONString(m));
+ }
+ tag = parse(m.keySet().stream()
+ .filter(s -> (!globalTagName.get().equals(s) && !IGNORE_TAGS.contains(s)))
+ .findFirst().get(), m);
+ } else {
+ collection = parse(COLLECTION, m);
+ shard = parse(SHARD, m);
+ if(m.get(REPLICA) == null){
+ throw new RuntimeException(StrUtils.formatString("'replica' is required in {0}", Utils.toJSONString(m)));
+ }
+ this.replica = parse(REPLICA, m);
+ if (replica.op == WILDCARD) throw new RuntimeException("replica val cannot be null" + Utils.toJSONString(m));
+ m.forEach((s, o) -> parseCondition(s, o));
+ }
+ if (tag == null)
+ throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
+
+ }
+
+ public boolean doesOverride(Clause that) {
+ return (collection.equals(that.collection) &&
+ tag.name.equals(that.tag.name));
+
+ }
+
+ public boolean isPerCollectiontag() {
+ return globalTag == null;
+ }
+
+ void parseCondition(String s, Object o) {
+ if (IGNORE_TAGS.contains(s)) return;
+ if (tag != null) {
+ throw new IllegalArgumentException("Only one tag other than collection, shard, replica is possible");
+ }
+ tag = parse(s, singletonMap(s, o));
+ }
+
++ private int compareTypes(Replica.Type t1, Replica.Type t2) {
++ if (t1 == null && t2 == null) return 0;
++ if (t1 != null && t2 == null) return -1;
++ if (t1 == null) return 1;
++ return 0;
++ }
++
+ @Override
+ public int compareTo(Clause that) {
- try {
- int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
- if (v != 0) return v;
- if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
- v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
- if (v == 0) {
- v = Long.compare((Long) this.replica.val, (Long) that.replica.val);
- v = this.replica.op == LESS_THAN ? v : v * -1;
- }
- return v;
- } else {
- return 0;
++ int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
++ if (v != 0) return v;
++ if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
++ v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
++ if (v == 0) {// higher the number of replicas , harder to satisfy
++ v = Long.compare((Long) this.replica.val, (Long) that.replica.val);
++ v = this.replica.op == LESS_THAN ? v : v * -1;
+ }
- } catch (NullPointerException e) {
- throw e;
++ if (v == 0) v = compareTypes(this.type, that.type);
++ return v;
++ } else {
++ return 0;
+ }
++
+ }
+
+ void addTags(List<String> params) {
+ if (globalTag != null && !params.contains(globalTag.name)) params.add(globalTag.name);
+ if (tag != null && !params.contains(tag.name)) params.add(tag.name);
+ }
+
+ public static class Condition {
+ final String name;
+ final Object val;
+ final Operand op;
+
+ Condition(String name, Object val, Operand op) {
+ this.name = name;
+ this.val = val;
+ this.op = op;
+ }
+
- TestStatus match(Row row) {
- return op.match(val, row.getVal(name));
- }
-
- TestStatus match(Object testVal) {
- return op.match(this.val, testVal);
- }
-
+ public boolean isPass(Object inputVal) {
++ if (inputVal instanceof ReplicaCount) inputVal = ((ReplicaCount) inputVal).getVal(type);
+ return op.match(val, validate(name, inputVal, false)) == PASS;
+ }
+
+ public boolean isPass(Row row) {
+ return op.match(val, row.getVal(name)) == PASS;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that instanceof Condition) {
+ Condition c = (Condition) that;
+ return Objects.equals(c.name, name) && Objects.equals(c.val, val) && c.op == op;
+ }
+ return false;
+ }
+
- public Integer delta(Object val) {
++ public Long delta(Object val) {
+ return op.delta(this.val, val);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Object getValue() {
+ return val;
+ }
-
++
+ public Operand getOperand() {
+ return op;
+ }
+ }
+
+ static Condition parse(String s, Map m) {
+ Object expectedVal = null;
+ Object val = m.get(s);
+ try {
+ String conditionName = s.trim();
+ Operand operand = null;
+ if (val == null) {
+ operand = WILDCARD;
+ expectedVal = Policy.ANY;
+ } else if (val instanceof String) {
+ String strVal = ((String) val).trim();
+ if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
+ else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
+ else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
+ else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
+ else operand = EQUAL;
+ expectedVal = validate(s, strVal.substring(EQUAL == operand || WILDCARD == operand ? 0 : 1), true);
+ } else if (val instanceof Number) {
+ operand = EQUAL;
+ expectedVal = validate(s, val, true);
+ }
+ return new Condition(conditionName, expectedVal, operand);
+
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid tag : " + s + ":" + val, e);
+ }
+ }
+
+ public class Violation implements MapWriter {
+ final String shard, coll, node;
+ final Object actualVal;
- final Integer delta;//how far is the actual value from the expected value
++ final Long delta;//how far is the actual value from the expected value
+ final Object tagKey;
+ private final int hash;
+
+
- private Violation(String coll, String shard, String node, Object actualVal, Integer delta, Object tagKey) {
++ private Violation(String coll, String shard, String node, Object actualVal, Long delta, Object tagKey) {
+ this.shard = shard;
+ this.coll = coll;
+ this.node = node;
+ this.delta = delta;
+ this.actualVal = actualVal;
+ this.tagKey = tagKey;
+ hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
+ }
+
+ public Clause getClause() {
+ return Clause.this;
+ }
+
+ @Override
+ public int hashCode() {
+ return hash;
+ }
+ //if the delta is lower , this violation is less serious
+ public boolean isLessSerious(Violation that) {
+ return that.delta != null && delta != null &&
+ Math.abs(delta) < Math.abs(that.delta);
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that instanceof Violation) {
+ Violation v = (Violation) that;
+ return Objects.equals(this.shard, v.shard) &&
+ Objects.equals(this.coll, v.coll) &&
+ Objects.equals(this.node, v.node) &&
+ Objects.equals(this.tagKey, v.tagKey)
+ ;
+ }
+ return false;
+ }
+
+ @Override
++ public String toString() {
++ return Utils.toJSONString(Utils.getDeepCopy(toMap(new LinkedHashMap<>()), 5));
++ }
++
++ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.putIfNotNull("collection", coll);
+ ew.putIfNotNull("shard", shard);
+ ew.putIfNotNull("node", node);
+ ew.putIfNotNull("tagKey", String.valueOf(tagKey));
+ ew.putIfNotNull("violation", (MapWriter) ew1 -> {
- ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
- String.valueOf(actualVal));
++ if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal);
++ else ew1.put(tag.name, String.valueOf(actualVal));
+ ew1.putIfNotNull("delta", delta);
+ });
+ ew.put("clause", getClause());
+ }
+ }
+
+
+ public List<Violation> test(List<Row> allRows) {
+ List<Violation> violations = new ArrayList<>();
+ if (isPerCollectiontag()) {
- Map<String, Map<String, Map<String, AtomicInteger>>> replicaCount = computeReplicaCounts(allRows);
- for (Map.Entry<String, Map<String, Map<String, AtomicInteger>>> e : replicaCount.entrySet()) {
++ Map<String, Map<String, Map<String, ReplicaCount>>> replicaCount = computeReplicaCounts(allRows);
++ for (Map.Entry<String, Map<String, Map<String, ReplicaCount>>> e : replicaCount.entrySet()) {
+ if (!collection.isPass(e.getKey())) continue;
- for (Map.Entry<String, Map<String, AtomicInteger>> shardVsCount : e.getValue().entrySet()) {
++ for (Map.Entry<String, Map<String, ReplicaCount>> shardVsCount : e.getValue().entrySet()) {
+ if (!shard.isPass(shardVsCount.getKey())) continue;
- for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
++ for (Map.Entry<String, ReplicaCount> counts : shardVsCount.getValue().entrySet()) {
+ if (!replica.isPass(counts.getValue())) {
+ violations.add(new Violation(
+ e.getKey(),
+ shardVsCount.getKey(),
+ tag.name.equals("node") ? counts.getKey() : null,
+ counts.getValue(),
+ replica.delta(counts.getValue()),
+ counts.getKey()
+ ));
+ }
+ }
+ }
+ }
+ } else {
+ for (Row r : allRows) {
+ if (!tag.isPass(r)) {
+ violations.add(new Violation(null, null, r.node, r.getVal(tag.name), tag.delta(r.getVal(tag.name)), null));
+ }
+ }
+ }
+ return violations;
+
+ }
+
+
- private Map<String, Map<String, Map<String, AtomicInteger>>> computeReplicaCounts(List<Row> allRows) {
- Map<String, Map<String, Map<String, AtomicInteger>>> collVsShardVsTagVsCount = new HashMap<>();
- for (Row row : allRows)
++ private Map<String, Map<String, Map<String, ReplicaCount>>> computeReplicaCounts(List<Row> allRows) {
++ Map<String, Map<String, Map<String, ReplicaCount>>> collVsShardVsTagVsCount = new HashMap<>();
++ for (Row row : allRows) {
+ for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
+ String collectionName = colls.getKey();
+ if (!collection.isPass(collectionName)) continue;
- collVsShardVsTagVsCount.putIfAbsent(collectionName, new HashMap<>());
- Map<String, Map<String, AtomicInteger>> collMap = collVsShardVsTagVsCount.get(collectionName);
++ Map<String, Map<String, ReplicaCount>> collMap = collVsShardVsTagVsCount.computeIfAbsent(collectionName, s -> new HashMap<>());
+ for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
+ String shardName = shards.getKey();
+ if (ANY.equals(shard.val)) shardName = ANY;
+ if (!shard.isPass(shardName)) break;
- collMap.putIfAbsent(shardName, new HashMap<>());
- Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
++ Map<String, ReplicaCount> tagVsCount = collMap.computeIfAbsent(shardName, s -> new HashMap<>());
+ Object tagVal = row.getVal(tag.name);
- tagVsCount.putIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", new AtomicInteger());
++ tagVsCount.computeIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
+ if (tag.isPass(tagVal)) {
- tagVsCount.get(String.valueOf(tagVal)).addAndGet(shards.getValue().size());
++ tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
++ }
+ }
+ }
+ }
+ return collVsShardVsTagVsCount;
+ }
+
+ public boolean isStrict() {
+ return strict;
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(original);
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (Map.Entry<String, Object> e : original.entrySet()) ew.put(e.getKey(), e.getValue());
+ }
+
+ enum TestStatus {
+ NOT_APPLICABLE, FAIL, PASS
+ }
+
- private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict"));
++ private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict", "type"));
+
+ static class ValidateInfo {
+ final Class type;
+ final Set<String> vals;
+ final Number min;
+ final Number max;
+
+
+ ValidateInfo(Class type, Set<String> vals, Number min, Number max) {
+ this.type = type;
+ this.vals = vals;
+ this.min = min;
- if(min != null && !type.isInstance(min)) throw new RuntimeException("wrong min value type, expected: " + type.getName() + " actual: " + min.getClass().getName());
++ if (min != null && !type.isInstance(min))
++ throw new RuntimeException("wrong min value type, expected: " + type.getName() + " actual: " + min.getClass().getName());
+ this.max = max;
- if(max != null && !type.isInstance(max)) throw new RuntimeException("wrong max value type, expected: " + type.getName() + " actual: " + max.getClass().getName());
++ if (max != null && !type.isInstance(max))
++ throw new RuntimeException("wrong max value type, expected: " + type.getName() + " actual: " + max.getClass().getName());
+ }
+ }
+
+
+ /**
- *
- * @param name name of the condition
- * @param val value of the condition
++ * @param name name of the condition
++ * @param val value of the condition
+ * @param isRuleVal is this provided in the rule
+ * @return actual validated value
+ */
+ public static Object validate(String name, Object val, boolean isRuleVal) {
+ if (val == null) return null;
+ ValidateInfo info = validatetypes.get(name);
+ if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = validatetypes.get("STRING");
+ if (info == null) throw new RuntimeException("Unknown type :" + name);
+ if (info.type == Double.class) {
+ Double num = parseDouble(name, val);
+ if (isRuleVal) {
+ if (info.min != null)
+ if (Double.compare(num, (Double) info.min) == -1)
+ throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+ if (info.max != null)
+ if (Double.compare(num, (Double) info.max) == 1)
+ throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+ }
+ return num;
+ } else if (info.type == Long.class) {
+ Long num = parseLong(name, val);
+ if (isRuleVal) {
+ if (info.min != null)
+ if (num < info.min.longValue())
+ throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+ if (info.max != null)
+ if (num > info.max.longValue())
+ throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+ }
+ return num;
+ } else if (info.type == String.class) {
+ if (isRuleVal && info.vals != null && !info.vals.contains(val))
+ throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(info.vals, ','));
+ return val;
+ } else {
+ throw new RuntimeException("Invalid type ");
+ }
+ }
+
+ public static Long parseLong(String name, Object val) {
+ if (val == null) return null;
+ if (val instanceof Long) return (Long) val;
+ Number num = null;
+ if (val instanceof String) {
+ try {
+ num = Long.parseLong(((String) val).trim());
+ } catch (NumberFormatException e) {
+ try {
+ num = Double.parseDouble((String) val);
+ } catch (NumberFormatException e1) {
+ throw new RuntimeException(name + ": " + val + "not a valid number", e);
+ }
+ }
+
+ } else if (val instanceof Number) {
+ num = (Number) val;
+ }
+
- if (num != null) {
++ if (num != null) {
+ return num.longValue();
+ }
+ throw new RuntimeException(name + ": " + val + "not a valid number");
+ }
+
+ public static Double parseDouble(String name, Object val) {
+ if (val == null) return null;
+ if (val instanceof Double) return (Double) val;
+ Number num = null;
+ if (val instanceof String) {
+ try {
+ num = Double.parseDouble((String) val);
+ } catch (NumberFormatException e) {
+ throw new RuntimeException(name + ": " + val + "not a valid number", e);
+ }
+
+ } else if (val instanceof Number) {
+ num = (Number) val;
+ }
+
- if (num != null) {
++ if (num != null) {
+ return num.doubleValue();
+ }
+ throw new RuntimeException(name + ": " + val + "not a valid number");
+ }
+
+ private static final Map<String, ValidateInfo> validatetypes = new HashMap<>();
+
+ static {
+ validatetypes.put("collection", new ValidateInfo(String.class, null, null, null));
+ validatetypes.put("shard", new ValidateInfo(String.class, null, null, null));
+ validatetypes.put("replica", new ValidateInfo(Long.class, null, 0L, null));
+ validatetypes.put(ImplicitSnitch.PORT, new ValidateInfo(Long.class, null, 1L, 65535L));
+ validatetypes.put(ImplicitSnitch.DISK, new ValidateInfo(Double.class, null, 0d, Double.MAX_VALUE));
+ validatetypes.put(ImplicitSnitch.NODEROLE, new ValidateInfo(String.class, Collections.singleton("overseer"), null, null));
+ validatetypes.put(ImplicitSnitch.CORES, new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));
+ validatetypes.put(ImplicitSnitch.SYSLOADAVG, new ValidateInfo(Double.class, null, 0d, 100d));
+ validatetypes.put(ImplicitSnitch.HEAPUSAGE, new ValidateInfo(Double.class, null, 0d, null));
+ validatetypes.put("NUMBER", new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));//generic number validation
+ validatetypes.put("STRING", new ValidateInfo(String.class, null, null, null));//generic string validation
+ validatetypes.put("node", new ValidateInfo(String.class, null, null, null));
+ for (String ip : ImplicitSnitch.IP_SNITCHES) validatetypes.put(ip, new ValidateInfo(Long.class, null, 0L, 255L));
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
index 0000000,e873625..58972af
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
@@@ -1,0 -1,52 +1,52 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import java.io.Closeable;
+ import java.io.IOException;
+ import java.util.Collection;
+ import java.util.List;
+ import java.util.Map;
+
+ public interface ClusterDataProvider extends Closeable {
+ /**Get the value of each tag for a given node
+ *
+ * @param node node name
+ * @param tags tag names
+ * @return a map of tag vs value
+ */
+ Map<String, Object> getNodeValues(String node, Collection<String> tags);
+
+ /**
+ * Get the details of each replica in a node. It attempts to fetch as much details about
+ * the replica as mentioned in the keys list. It is not necessary to give al details
+ * <p>
+ * the format is {collection:shard :[{replicadetails}]}
+ */
- Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
++ Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
+
+ Collection<String> getNodes();
+
+ /**Get the collection-specific policy
+ */
+ String getPolicyNameByCollection(String coll);
+
+ @Override
+ default void close() throws IOException {
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
index 0000000,bf9c284..a068253
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@@ -1,0 -1,83 +1,83 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import java.util.List;
+
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+ import org.apache.solr.common.util.Pair;
+
+ public class MoveReplicaSuggester extends Suggester {
+
+ @Override
+ SolrRequest init() {
+ SolrRequest operation = tryEachNode(true);
+ if (operation == null) operation = tryEachNode(false);
+ return operation;
+ }
+
+ SolrRequest tryEachNode(boolean strict) {
+ //iterate through elements and identify the least loaded
+ List<Clause.Violation> leastSeriousViolation = null;
+ Integer targetNodeIndex = null;
+ Integer fromNodeIndex = null;
+ ReplicaInfo fromReplicaInfo = null;
+ for (Pair<ReplicaInfo, Row> fromReplica : getValidReplicas(true, true, -1)) {
+ Row fromRow = fromReplica.second();
+ ReplicaInfo replicaInfo = fromReplica.first();
+ String coll = replicaInfo.collection;
+ String shard = replicaInfo.shard;
- Pair<Row, ReplicaInfo> pair = fromRow.removeReplica(coll, shard);
++ Pair<Row, ReplicaInfo> pair = fromRow.removeReplica(coll, shard, replicaInfo.type);
+ Row tmpRow = pair.first();
+ if (tmpRow == null) {
+ //no such replica available
+ continue;
+ }
+ tmpRow.violations.clear();
+
+ final int i = getMatrix().indexOf(fromRow);
+ for (int j = getMatrix().size() - 1; j > i; j--) {
+ Row targetRow = getMatrix().get(j);
++ if(!targetRow.isLive) continue;
+ if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
- targetRow = targetRow.addReplica(coll, shard);
++ targetRow = targetRow.addReplica(coll, shard, replicaInfo.type);
+ targetRow.violations.clear();
+ List<Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getModifiedMatrix(getMatrix(), tmpRow, i), targetRow, j));
+ if (!containsNewErrors(errs) && isLessSerious(errs, leastSeriousViolation)) {
+ leastSeriousViolation = errs;
+ targetNodeIndex = j;
+ fromNodeIndex = i;
+ fromReplicaInfo = replicaInfo;
+ }
+ }
+ }
+ if (targetNodeIndex != null && fromNodeIndex != null) {
- getMatrix().set(fromNodeIndex, getMatrix().get(fromNodeIndex).removeReplica(fromReplicaInfo.collection, fromReplicaInfo.shard).first());
- getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(fromReplicaInfo.collection, fromReplicaInfo.shard));
++ getMatrix().set(fromNodeIndex, getMatrix().get(fromNodeIndex).removeReplica(fromReplicaInfo.collection, fromReplicaInfo.shard, fromReplicaInfo.type).first());
++ getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(fromReplicaInfo.collection, fromReplicaInfo.shard, fromReplicaInfo.type));
+ return new CollectionAdminRequest.MoveReplica(
+ fromReplicaInfo.collection,
+ fromReplicaInfo.name,
+ getMatrix().get(targetNodeIndex).node);
+ }
+ return null;
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
index 0000000,e012718..6522cb2
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
@@@ -1,0 -1,123 +1,124 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.FAIL;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.NOT_APPLICABLE;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+
+ import java.util.Objects;
+
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus;
++import org.apache.solr.common.params.CoreAdminParams;
+
+
+ public enum Operand {
+ WILDCARD(ANY, Integer.MAX_VALUE) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return testVal == null ? NOT_APPLICABLE : PASS;
+ }
+
+ },
+ EQUAL("", 0) {
+ @Override
- public int _delta(int expected, int actual) {
++ public long _delta(long expected, long actual) {
+ return expected - actual;
+ }
+ },
+ NOT_EQUAL("!", 2) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
+ }
+
+ @Override
- public int _delta(int expected, int actual) {
++ public long _delta(long expected, long actual) {
+ return expected - actual;
+ }
+
+ },
+ GREATER_THAN(">", 1) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ if (testVal == null) return NOT_APPLICABLE;
+ if (ruleVal instanceof Double) {
+ return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == 1 ? PASS : FAIL;
+ }
+ return getLong(testVal) > getLong(ruleVal) ? PASS: FAIL ;
+ }
+
+ @Override
- protected int _delta(int expected, int actual) {
++ protected long _delta(long expected, long actual) {
+ return actual > expected ? 0 : (expected + 1) - actual;
+ }
+ },
+ LESS_THAN("<", 2) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ if (testVal == null) return NOT_APPLICABLE;
+ if (ruleVal instanceof Double) {
+ return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == -1 ? PASS : FAIL;
+ }
+ return getLong(testVal) < getLong(ruleVal) ? PASS: FAIL ;
+ }
+
+ @Override
- protected int _delta(int expected, int actual) {
++ protected long _delta(long expected, long actual) {
+ return actual < expected ? 0 : (expected ) - actual;
+ }
+
+ };
+ public final String operand;
+ final int priority;
+
+ Operand(String val, int priority) {
+ this.operand = val;
+ this.priority = priority;
+ }
+
+ public String toStr(Object expectedVal) {
+ return operand + expectedVal.toString();
+ }
+
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return Objects.equals(ruleVal, testVal) ? PASS : FAIL;
+ }
+
+ Long getLong(Object o) {
+ if (o instanceof Long) return (Long) o;
+ if(o instanceof Number ) return ((Number) o).longValue();
+ return Long.parseLong(String.valueOf(o));
+
+ }
+
- public Integer delta(Object expected, Object actual) {
++ public Long delta(Object expected, Object actual) {
+ try {
- Integer expectedInt = Integer.parseInt(String.valueOf(expected));
- Integer actualInt = Integer.parseInt(String.valueOf(actual));
++ Long expectedInt = (Long) Clause.validate(CoreAdminParams.REPLICA, expected, false);
++ Long actualInt = (Long) Clause.validate(CoreAdminParams.REPLICA, actual, false);
+ return _delta(expectedInt, actualInt);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
- protected int _delta(int expected, int actual) {
++ protected long _delta(long expected, long actual) {
+ return 0;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 0000000,ccb0dee..451c514
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@@ -1,0 -1,521 +1,499 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.EnumMap;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.LinkedHashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.Set;
+ import java.util.SortedSet;
+ import java.util.TreeSet;
+ import java.util.function.Supplier;
+ import java.util.stream.Collectors;
+
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+ import org.apache.solr.common.IteratorWriter;
+ import org.apache.solr.common.MapWriter;
+ import org.apache.solr.common.params.CollectionParams.CollectionAction;
+ import org.apache.solr.common.util.Pair;
+ import org.apache.solr.common.util.StrUtils;
+ import org.apache.solr.common.util.Utils;
+
+ import static java.util.Collections.emptyList;
+ import static java.util.Collections.emptyMap;
+ import static java.util.stream.Collectors.toList;
+
+ /*The class that reads, parses and applies policies specified in
+ * autoscaling.json
+ *
+ * Create one instance of this class per unique autoscaling.json.
+ * This is immutable and is thread-safe
+ *
+ * Create a fresh new session for each use
+ *
+ */
+ public class Policy implements MapWriter {
+ public static final String POLICY = "policy";
+ public static final String EACH = "#EACH";
+ public static final String ANY = "#ANY";
+ public static final String CLUSTER_POLICY = "cluster-policy";
+ public static final String CLUSTER_PREFERENCE = "cluster-preferences";
+ public static final Set<String> GLOBAL_ONLY_TAGS = Collections.singleton("cores");
+ final Map<String, List<Clause>> policies = new HashMap<>();
+ final List<Clause> clusterPolicy;
+ final List<Preference> clusterPreferences;
+ final List<String> params;
+
+
+ public Policy(Map<String, Object> jsonMap) {
+
+ clusterPreferences = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_PREFERENCE, emptyList())).stream()
+ .map(Preference::new)
+ .collect(toList());
+ for (int i = 0; i < clusterPreferences.size() - 1; i++) {
+ Preference preference = clusterPreferences.get(i);
+ preference.next = clusterPreferences.get(i + 1);
+ }
+ if (clusterPreferences.isEmpty()) {
+ clusterPreferences.add(new Preference((Map<String, Object>) Utils.fromJSONString("{minimize : cores, precision:1}")));
+ }
+ SortedSet<String> paramsOfInterest = new TreeSet<>();
+ for (Preference preference : clusterPreferences) {
+ if (paramsOfInterest.contains(preference.name.name())) {
+ throw new RuntimeException(preference.name + " is repeated");
+ }
+ paramsOfInterest.add(preference.name.toString());
+ }
+ this.params = new ArrayList<>(paramsOfInterest);
+
+ clusterPolicy = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_POLICY, emptyList())).stream()
+ .map(Clause::new)
+ .filter(clause -> {
+ clause.addTags(params);
+ return true;
+ })
+ .collect(Collectors.toList());
+
+ ((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault("policies", emptyMap())).forEach((s, l1) ->
+ this.policies.put(s, l1.stream()
+ .map(Clause::new)
+ .filter(clause -> {
+ if (!clause.isPerCollectiontag())
+ throw new RuntimeException(clause.globalTag.name + " is only allowed in 'cluster-policy'");
+ clause.addTags(params);
+ return true;
+ })
+ .sorted()
+ .collect(toList())));
+ }
+
+ public List<Clause> getClusterPolicy() {
+ return clusterPolicy;
+ }
+
+ public List<Preference> getClusterPreferences() {
+ return clusterPreferences;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ if (!policies.isEmpty()) {
+ ew.put("policies", (MapWriter) ew1 -> {
+ for (Map.Entry<String, List<Clause>> e : policies.entrySet()) {
+ ew1.put(e.getKey(), e.getValue());
+ }
+ });
+ }
+ if (!clusterPreferences.isEmpty()) {
+ ew.put("preferences", (IteratorWriter) iw -> {
+ for (Preference p : clusterPreferences) iw.add(p);
+ });
+ }
+
+ }
+
+ /*This stores the logical state of the system, given a policy and
+ * a cluster state.
+ *
+ */
+ public class Session implements MapWriter {
+ final List<String> nodes;
+ final ClusterDataProvider dataProvider;
+ final List<Row> matrix;
+ Set<String> collections = new HashSet<>();
+ List<Clause> expandedClauses;
+ List<Violation> violations = new ArrayList<>();
+
+ private Session(List<String> nodes, ClusterDataProvider dataProvider,
+ List<Row> matrix, List<Clause> expandedClauses) {
+ this.nodes = nodes;
+ this.dataProvider = dataProvider;
+ this.matrix = matrix;
+ this.expandedClauses = expandedClauses;
+ }
+
+ Session(ClusterDataProvider dataProvider) {
+ this.nodes = new ArrayList<>(dataProvider.getNodes());
+ this.dataProvider = dataProvider;
+ for (String node : nodes) {
+ collections.addAll(dataProvider.getReplicaInfo(node, Collections.emptyList()).keySet());
+ }
+
+ expandedClauses = clusterPolicy.stream()
+ .filter(clause -> !clause.isPerCollectiontag())
+ .collect(Collectors.toList());
+
+ for (String c : collections) {
+ addClausesForCollection(dataProvider, c);
+ }
+
+ Collections.sort(expandedClauses);
+
+ matrix = new ArrayList<>(nodes.size());
+ for (String node : nodes) matrix.add(new Row(node, params, dataProvider));
+ applyRules();
+ }
+
+ private void addClausesForCollection(ClusterDataProvider dataProvider, String c) {
+ String p = dataProvider.getPolicyNameByCollection(c);
+ if (p != null) {
+ List<Clause> perCollPolicy = policies.get(p);
+ if (perCollPolicy == null)
+ throw new RuntimeException(StrUtils.formatString("Policy for collection {0} is {1} . It does not exist", c, p));
+ }
+ expandedClauses.addAll(mergePolicies(c, policies.getOrDefault(p, emptyList()), clusterPolicy));
+ }
+
+ Session copy() {
+ return new Session(nodes, dataProvider, getMatrixCopy(), expandedClauses);
+ }
+
+ List<Row> getMatrixCopy() {
+ return matrix.stream()
+ .map(Row::copy)
+ .collect(Collectors.toList());
+ }
+
+ Policy getPolicy() {
+ return Policy.this;
+
+ }
+
+ /**
+ * Apply the preferences and conditions
+ */
+ private void applyRules() {
+ if (!clusterPreferences.isEmpty()) {
+ //this is to set the approximate value according to the precision
+ ArrayList<Row> tmpMatrix = new ArrayList<>(matrix);
+ for (Preference p : clusterPreferences) {
+ Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false));
+ p.setApproxVal(tmpMatrix);
+ }
+ //approximate values are set now. Let's do recursive sorting
+ Collections.sort(matrix, (Row r1, Row r2) -> {
+ int result = clusterPreferences.get(0).compare(r1, r2, true);
+ if (result == 0) result = clusterPreferences.get(0).compare(r1, r2, false);
+ return result;
+ });
+ }
+
+ for (Clause clause : expandedClauses) {
+ List<Violation> errs = clause.test(matrix);
+ violations.addAll(errs);
+ }
+ }
+
+ public List<Violation> getViolations() {
+ return violations;
+ }
+
+ public Suggester getSuggester(CollectionAction action) {
+ Suggester op = ops.get(action).get();
+ if (op == null) throw new UnsupportedOperationException(action.toString() + "is not supported");
+ op._init(this);
+ return op;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (int i = 0; i < matrix.size(); i++) {
+ Row row = matrix.get(i);
+ ew.put(row.node, row);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(toMap(new LinkedHashMap<>()));
+ }
+
+ public List<Row> getSorted() {
+ return Collections.unmodifiableList(matrix);
+ }
+ }
+
+
+ public Session createSession(ClusterDataProvider dataProvider) {
+ return new Session(dataProvider);
+ }
+
+ public enum SortParam {
+ freedisk(0, Integer.MAX_VALUE), cores(0, Integer.MAX_VALUE), heapUsage(0, Integer.MAX_VALUE), sysLoadAvg(0, 100);
+
+ public final int min,max;
+
+ SortParam(int min, int max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ static SortParam get(String m) {
+ for (SortParam p : values()) if (p.name().equals(m)) return p;
+ throw new RuntimeException(StrUtils.formatString("Invalid sort {0} Sort must be on one of these {1}", m, Arrays.asList(values())));
+ }
+ }
+
+ enum Sort {
+ maximize(1), minimize(-1);
+ final int sortval;
+
+ Sort(int i) {
+ sortval = i;
+ }
+
+ static Sort get(Map<String, Object> m) {
+ if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) {
+ throw new RuntimeException("Cannot have both 'maximize' and 'minimize'");
+ }
+ if (m.containsKey(maximize.name())) return maximize;
+ if (m.containsKey(minimize.name())) return minimize;
+ throw new RuntimeException("must have either 'maximize' or 'minimize'");
+ }
+ }
+
+
- public static class ReplicaInfo implements MapWriter {
- final String name;
- String core, collection, shard;
- Map<String, Object> variables;
-
- public ReplicaInfo(String name, String coll, String shard, Map<String, Object> vals) {
- this.name = name;
- this.variables = vals;
- this.collection = coll;
- this.shard = shard;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(name, variables);
- }
-
- public String getCore() {
- return core;
- }
-
- public String getCollection() {
- return collection;
- }
-
- public String getShard() {
- return shard;
- }
- }
-
-
+ /* A suggester is capable of suggesting a collection operation
+ * given a particular session. Before it suggests a new operation,
+ * it ensures that ,
+ * a) load is reduced on the most loaded node
+ * b) it causes no new violations
+ *
+ */
+ public static abstract class Suggester {
+ protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
+ Policy.Session session;
+ SolrRequest operation;
+ protected List<Violation> originalViolations = new ArrayList<>();
+ private boolean isInitialized = false;
+
+ private void _init(Session session) {
+ this.session = session.copy();
+ }
+
+ public Suggester hint(Hint hint, Object value) {
+ if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+ ((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).add(value);
+ } else {
- hints.put(hint, value);
++ hints.put(hint, value == null ? null : String.valueOf(value));
+ }
+ return this;
+ }
+
+ abstract SolrRequest init();
+
+
+ public SolrRequest getOperation() {
+ if (!isInitialized) {
+ String coll = (String) hints.get(Hint.COLL);
+ String shard = (String) hints.get(Hint.SHARD);
+ // if this is not a known collection from the existing clusterstate,
+ // then add it
+ if (session.matrix.stream().noneMatch(row -> row.collectionVsShardVsReplicas.containsKey(coll))) {
+ session.addClausesForCollection(session.dataProvider, coll);
+ Collections.sort(session.expandedClauses);
+ }
+ if (coll != null) {
+ for (Row row : session.matrix) {
+ if (!row.collectionVsShardVsReplicas.containsKey(coll)) row.collectionVsShardVsReplicas.put(coll, new HashMap<>());
+ if (shard != null) {
+ Map<String, List<ReplicaInfo>> shardInfo = row.collectionVsShardVsReplicas.get(coll);
+ if (!shardInfo.containsKey(shard)) shardInfo.put(shard, new ArrayList<>());
+ }
+ }
+ }
++ Set<String> srcNodes = (Set<String>) hints.get(Hint.SRC_NODE);
++ if (srcNodes != null && !srcNodes.isEmpty()) {
++ // the source node is dead so live nodes may not have it
++ for (String srcNode : srcNodes) {
++ if(session.matrix.stream().noneMatch(row -> row.node.equals(srcNode)))
++ session.matrix.add(new Row(srcNode, session.getPolicy().params, session.dataProvider));
++ }
++ }
+ session.applyRules();
+ originalViolations.addAll(session.getViolations());
+ this.operation = init();
+ isInitialized = true;
+ }
+ return operation;
+ }
+
+ public Session getSession() {
+ return session;
+ }
+
+ List<Row> getMatrix() {
+ return session.matrix;
+
+ }
+
+ //check if the fresh set of violations is less serious than the last set of violations
+ boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
+ if (old == null || fresh.size() < old.size()) return true;
+ if (fresh.size() == old.size()) {
+ for (int i = 0; i < fresh.size(); i++) {
+ Violation freshViolation = fresh.get(i);
+ Violation oldViolation = null;
+ for (Violation v : old) {
+ if (v.equals(freshViolation)) oldViolation = v;
+ }
+ if (oldViolation != null && freshViolation.isLessSerious(oldViolation)) return true;
+ }
+ }
+ return false;
+ }
+
+ boolean containsNewErrors(List<Violation> violations) {
+ for (Violation v : violations) {
+ int idx = originalViolations.indexOf(v);
+ if (idx < 0 || originalViolations.get(idx).isLessSerious(v)) return true;
+ }
+ return false;
+ }
+
+ List<Pair<ReplicaInfo, Row>> getValidReplicas(boolean sortDesc, boolean isSource, int until) {
- List<Pair<Policy.ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
++ List<Pair<ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
+
+ if (sortDesc) {
+ if (until == -1) until = getMatrix().size();
+ for (int i = 0; i < until; i++) addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+ } else {
+ if (until == -1) until = 0;
+ for (int i = getMatrix().size() - 1; i >= until; i--)
+ addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+ }
+ return allPossibleReplicas;
+ }
+
- void addReplicaToList(Row r, boolean isSource, List<Pair<Policy.ReplicaInfo, Row>> replicaList) {
++ void addReplicaToList(Row r, boolean isSource, List<Pair<ReplicaInfo, Row>> replicaList) {
+ if (!isAllowed(r.node, isSource ? Hint.SRC_NODE : Hint.TARGET_NODE)) return;
- for (Map.Entry<String, Map<String, List<Policy.ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
++ for (Map.Entry<String, Map<String, List<ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
+ if (!isAllowed(e.getKey(), Hint.COLL)) continue;
- for (Map.Entry<String, List<Policy.ReplicaInfo>> shard : e.getValue().entrySet()) {
- if (!isAllowed(e.getKey(), Hint.SHARD)) continue;
++ for (Map.Entry<String, List<ReplicaInfo>> shard : e.getValue().entrySet()) {
++ if (!isAllowed(e.getKey(), Hint.SHARD)) continue;//todo fix
++ if(shard.getValue() == null || shard.getValue().isEmpty()) continue;
+ replicaList.add(new Pair<>(shard.getValue().get(0), r));
+ }
+ }
+ }
+
+ protected List<Violation> testChangedMatrix(boolean strict, List<Row> rows) {
+ List<Violation> errors = new ArrayList<>();
+ for (Clause clause : session.expandedClauses) {
+ if (strict || clause.strict) {
+ List<Violation> errs = clause.test(rows);
+ if (!errs.isEmpty()) {
+ errors.addAll(errs);
+ }
+ }
+ }
+ return errors;
+ }
+
+ ArrayList<Row> getModifiedMatrix(List<Row> matrix, Row tmpRow, int i) {
+ ArrayList<Row> copy = new ArrayList<>(matrix);
+ copy.set(i, tmpRow);
+ return copy;
+ }
+
+ protected boolean isAllowed(Object v, Hint hint) {
+ Object hintVal = hints.get(hint);
+ if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+ Set set = (Set) hintVal;
+ return set == null || set.contains(v);
+ } else {
+ return hintVal == null || Objects.equals(v, hintVal);
+ }
+ }
+
+ public enum Hint {
- COLL, SHARD, SRC_NODE, TARGET_NODE
++ COLL, SHARD, SRC_NODE, TARGET_NODE, REPLICATYPE
+ }
+
+
+ }
+
+ public static List<Clause> mergePolicies(String coll,
+ List<Clause> collPolicy,
+ List<Clause> globalPolicy) {
+
+ List<Clause> merged = insertColl(coll, collPolicy);
+ List<Clause> global = insertColl(coll, globalPolicy);
+ merged.addAll(global.stream()
+ .filter(clusterPolicyClause -> merged.stream().noneMatch(perCollPolicy -> perCollPolicy.doesOverride(clusterPolicyClause)))
+ .collect(Collectors.toList()));
+ return merged;
+ }
+
+ /**
+ * Insert the collection name into the clauses where collection is not specified
+ */
+ static List<Clause> insertColl(String coll, Collection<Clause> conditions) {
+ return conditions.stream()
+ .filter(Clause::isPerCollectiontag)
+ .map(clause -> {
+ Map<String, Object> copy = new LinkedHashMap<>(clause.original);
+ if (!copy.containsKey("collection")) copy.put("collection", coll);
+ return new Clause(copy);
+ })
+ .filter(it -> (it.collection.isPass(coll)))
+ .collect(Collectors.toList());
+
+ }
+
+ private static final Map<CollectionAction, Supplier<Suggester>> ops = new HashMap<>();
+
+ static {
+ ops.put(CollectionAction.ADDREPLICA, () -> new AddReplicaSuggester());
+ ops.put(CollectionAction.MOVEREPLICA, () -> new MoveReplicaSuggester());
+ }
+
+ public Map<String, List<Clause>> getPolicies() {
+ return policies;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 0000000,deea175..fe6c80b
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@@ -1,0 -1,96 +1,96 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
+ import org.apache.solr.common.SolrException;
+ import org.apache.solr.common.params.CoreAdminParams;
+ import org.apache.solr.common.util.Utils;
+
+ import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+
+ public class PolicyHelper {
+ public static Map<String, List<String>> getReplicaLocations(String collName, Map<String, Object> autoScalingJson,
+ ClusterDataProvider cdp,
+ Map<String, String> optionalPolicyMapping,
+ List<String> shardNames,
+ int repFactor,
+ List<String> nodesList) {
+ Map<String, List<String>> positionMapping = new HashMap<>();
+ for (String shardName : shardNames) positionMapping.put(shardName, new ArrayList<>(repFactor));
+ if (optionalPolicyMapping != null) {
+ final ClusterDataProvider delegate = cdp;
+ cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return delegate.getNodeValues(node, tags);
+ }
+
+ @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return delegate.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return delegate.getNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return optionalPolicyMapping.containsKey(coll) ?
+ optionalPolicyMapping.get(coll) :
+ delegate.getPolicyNameByCollection(coll);
+ }
+ };
+
+ }
+
+
+ Policy policy = new Policy(autoScalingJson);
+ Policy.Session session = policy.createSession(cdp);
+ for (String shardName : shardNames) {
+ for (int i = 0; i < repFactor; i++) {
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, collName)
+ .hint(Hint.SHARD, shardName);
+ if (nodesList != null) {
+ for (String nodeName : nodesList) {
+ suggester = suggester.hint(Hint.TARGET_NODE, nodeName);
+ }
+ }
+ SolrRequest op = suggester.getOperation();
+ if (op == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(Utils.getDeepCopy(session.expandedClauses, 4, true)));
+ }
+ session = suggester.getSession();
+ positionMapping.get(shardName).add(op.getParams().get(CoreAdminParams.NODE));
+ }
+ }
+
+ return positionMapping;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
index 0000000,bb45628..43fec6b
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
@@@ -1,0 -1,89 +1,92 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.solr.common.MapWriter;
+ import org.apache.solr.common.util.StrUtils;
+ import org.apache.solr.common.util.Utils;
+
+ public class Preference implements MapWriter {
+ final Policy.SortParam name;
+ Integer precision;
+ final Policy.Sort sort;
+ Preference next;
+ public int idx;
+ private final Map original;
+
+ Preference(Map<String, Object> m) {
+ this.original = Utils.getDeepCopy(m,3);
+ sort = Policy.Sort.get(m);
+ name = Policy.SortParam.get(m.get(sort.name()).toString());
+ Object p = m.getOrDefault("precision", 0);
+ precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
+ if (precision < 0) {
+ throw new RuntimeException("precision must be a positive value ");
+ }
+ if(precision< name.min || precision> name.max){
+ throw new RuntimeException(StrUtils.formatString("invalid precision value {0} must lie between {1} and {1}",
+ precision, name.min, name.max ) );
+ }
+
+ }
+
+ // there are 2 modes of compare.
+ // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
+ // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
+ int compare(Row r1, Row r2, boolean useApprox) {
++ if (!r1.isLive && !r2.isLive) return 0;
++ if (!r1.isLive) return -1;
++ if (!r2.isLive) return 1;
+ Object o1 = useApprox ? r1.cells[idx].approxVal : r1.cells[idx].val;
+ Object o2 = useApprox ? r2.cells[idx].approxVal : r2.cells[idx].val;
+ int result = 0;
+ if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
+ else if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
+ else if (!o1.getClass().getName().equals(o2.getClass().getName())) {
+ throw new RuntimeException("Unable to compare " + o1 + " of type: " + o1.getClass().getName() + " from " + r1.cells[idx].toString() + " and " + o2 + " of type: " + o2.getClass().getName() + " from " + r2.cells[idx].toString());
+ }
+ return result == 0 ? (next == null ? 0 : next.compare(r1, r2, useApprox)) : sort.sortval * result;
+ }
+
+ //sets the new value according to precision in val_
+ void setApproxVal(List<Row> tmpMatrix) {
+ Object prevVal = null;
+ for (Row row : tmpMatrix) {
+ prevVal = row.cells[idx].approxVal =
+ (prevVal == null || Double.compare(Math.abs(((Number) prevVal).doubleValue() - ((Number) row.cells[idx].val).doubleValue()), precision) > 0) ?
+ row.cells[idx].val :
+ prevVal;
+ }
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (Object o : original.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ ew.put(String.valueOf(e.getKey()), e.getValue());
+ }
+ }
+
+ public Policy.SortParam getName() {
+ return name;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 0000000,76c8c57..90559c1
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@@ -1,0 -1,120 +1,140 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
++import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Random;
+
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+ import org.apache.solr.common.IteratorWriter;
+ import org.apache.solr.common.MapWriter;
++import org.apache.solr.common.cloud.Replica;
+ import org.apache.solr.common.util.Pair;
+ import org.apache.solr.common.util.Utils;
+
+ import static org.apache.solr.common.params.CoreAdminParams.NODE;
+
+
+ public class Row implements MapWriter {
+ public final String node;
+ final Cell[] cells;
+ public Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas;
+ List<Clause> violations = new ArrayList<>();
+ boolean anyValueMissing = false;
++ boolean isLive = true;
+
+ public Row(String node, List<String> params, ClusterDataProvider dataProvider) {
+ collectionVsShardVsReplicas = dataProvider.getReplicaInfo(node, params);
+ if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>();
+ this.node = node;
+ cells = new Cell[params.size()];
- Map<String, Object> vals = dataProvider.getNodeValues(node, params);
++ isLive = dataProvider.getNodes().contains(node);
++ Map<String, Object> vals = isLive ? dataProvider.getNodeValues(node, params) : Collections.emptyMap();
+ for (int i = 0; i < params.size(); i++) {
+ String s = params.get(i);
+ cells[i] = new Cell(i, s, Clause.validate(s,vals.get(s), false));
+ if (NODE.equals(s)) cells[i].val = node;
+ if (cells[i].val == null) anyValueMissing = true;
+ }
+ }
+
- public Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, List<Clause> violations) {
++ public Row(String node, Cell[] cells, boolean anyValueMissing, Map<String,
++ Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, List<Clause> violations, boolean isLive) {
+ this.node = node;
++ this.isLive = isLive;
+ this.cells = new Cell[cells.length];
+ for (int i = 0; i < this.cells.length; i++) {
+ this.cells[i] = cells[i].copy();
+
+ }
+ this.anyValueMissing = anyValueMissing;
+ this.collectionVsShardVsReplicas = collectionVsShardVsReplicas;
+ this.violations = violations;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(node, (IteratorWriter) iw -> {
+ iw.add((MapWriter) e -> e.put("replicas", collectionVsShardVsReplicas));
+ for (Cell cell : cells) iw.add(cell);
+ });
+ }
+
+ Row copy() {
- return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(collectionVsShardVsReplicas, 3), new ArrayList<>(violations));
++ return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(collectionVsShardVsReplicas, 3), new ArrayList<>(violations), isLive);
+ }
+
+ Object getVal(String name) {
+ for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return node;
+ }
+
+ // this adds a replica to the replica info
- public Row addReplica(String coll, String shard) {
++ public Row addReplica(String coll, String shard, Replica.Type type) {
+ Row row = copy();
+ Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
+ List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
- replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, new HashMap<>()));
++ replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, type, new HashMap<>()));
+ for (Cell cell : row.cells) {
- if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() + 1;
++ if (cell.name.equals("cores")) {
++ cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() + 1;
++ }
+ }
+ return row;
+
+ }
+
- public Pair<Row, ReplicaInfo> removeReplica(String coll, String shard) {
++ public Pair<Row, ReplicaInfo> removeReplica(String coll, String shard, Replica.Type type) {
+ Row row = copy();
+ Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.get(coll);
+ if (c == null) return null;
- List<ReplicaInfo> s = c.get(shard);
- if (s == null || s.isEmpty()) return null;
++ List<ReplicaInfo> r = c.get(shard);
++ if (r == null) return null;
++ int idx = -1;
++ for (int i = 0; i < r.size(); i++) {
++ ReplicaInfo info = r.get(i);
++ if (type == null || info.type == type) {
++ idx = i;
++ break;
++ }
++ }
++ if(idx == -1) return null;
++
+ for (Cell cell : row.cells) {
- if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() -1;
++ if (cell.name.equals("cores")) {
++ cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() - 1;
++ }
+ }
- return new Pair(row, s.remove(0));
++ return new Pair(row, r.remove(idx));
+
+ }
+
+ public Cell[] getCells() {
+ return cells;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------
[14/18] lucene-solr:feature/autoscaling: Merge branch 'master' into
feature/autoscaling
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 8247524,a036c8d..d875593
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@@ -31,11 -31,16 +31,11 @@@ import java.util.Map
import com.google.common.collect.ImmutableList;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.autoscaling.Cell;
-import org.apache.solr.client.solrj.cloud.autoscaling.Clause;
-import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
-import org.apache.solr.client.solrj.cloud.autoscaling.Operand;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
-import org.apache.solr.client.solrj.cloud.autoscaling.Row;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
- import org.apache.solr.cloud.autoscaling.Clause.Violation;
- import org.apache.solr.cloud.autoscaling.Policy.Suggester.Hint;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.Utils;
@@@ -188,333 -195,15 +188,333 @@@ public class TestPolicy extends SolrTes
assertFalse(c.tag.isPass("12.6"));
assertFalse(c.tag.isPass(12.6d));
}
-
+
+ public void testNodeLost() {
+ String dataproviderdata = " {'liveNodes':[" +
+ " '127.0.0.1:65417_solr'," +
+ " '127.0.0.1:65434_solr']," +
+ " 'replicaInfo':{" +
+ " '127.0.0.1:65427_solr':{'testNodeLost':{'shard1':[{'core_node2':{type: NRT}}]}}," +
+ " '127.0.0.1:65417_solr':{'testNodeLost':{'shard1':[{'core_node1':{type: NRT}}]}}," +
+ " '127.0.0.1:65434_solr':{}}," +
+ " 'nodeValues':{" +
+ " '127.0.0.1:65417_solr':{" +
+ " 'node':'127.0.0.1:65417_solr'," +
+ " 'cores':1," +
+ " 'freedisk':884.7097854614258}," +
+ " '127.0.0.1:65434_solr':{" +
+ " 'node':'127.0.0.1:65434_solr'," +
+ " 'cores':0," +
+ " 'freedisk':884.7097854614258}}}";
+ /* String stateJson = "{'testNodeLost':{" +
+ " 'pullReplicas':'0'," +
+ " 'replicationFactor':'2'," +
+ " 'router':{'name':'compositeId'}," +
+ " 'maxShardsPerNode':'1'," +
+ " 'autoAddReplicas':'false'," +
+ " 'nrtReplicas':'2'," +
+ " 'tlogReplicas':'0'," +
+ " 'shards':{'shard1':{" +
+ " 'range':'80000000-7fffffff'," +
+ " 'state':'active'," +
+ " 'replicas':{" +
+ " 'core_node1':{" +
+ " 'core':'testNodeLost_shard1_replica_n1'," +
+ " 'base_url':'http://127.0.0.1:65417/solr'," +
+ " 'node_name':'127.0.0.1:65417_solr'," +
+ " 'state':'active'," +
+ " 'type':'NRT'," +
+ " 'leader':'true'}," +
+ " 'core_node2':{" +
+ " 'core':'testNodeLost_shard1_replica_n2'," +
+ " 'base_url':'http://127.0.0.1:65427/solr'," +
+ " 'node_name':'127.0.0.1:65427_solr'," +
+ " 'state':'down'," +
+ " 'type':'NRT'}}}}}}";*/
+
+ String autoScalingjson = "{" +
+ " 'cluster-policy':[" +
+ " {" +
+ " 'cores':'<10'," +
+ " 'node':'#ANY'}," +
+ " {" +
+ " 'replica':'<2'," +
+ " 'shard':'#EACH'," +
+ " 'node':'#ANY'}," +
+ " {" +
+ " 'nodeRole':'overseer'," +
+ " 'replica':0}]," +
+ " 'cluster-preferences':[" +
+ " {" +
+ " 'minimize':'cores'," +
+ " 'precision':3}," +
+ " {" +
+ " 'maximize':'freedisk'," +
+ " 'precision':100}]}";
-
++
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
+ Policy.Session session = policy.createSession(dataProviderWithData(dataproviderdata));
+ SolrRequest op = session.getSuggester(MOVEREPLICA).hint(Hint.SRC_NODE, "127.0.0.1:65427_solr").getOperation();
+ assertNotNull(op);
+ assertEquals( "127.0.0.1:65434_solr",op.getParams().get("targetNode") );
+ }
+
+ public void testNodeLostMultipleReplica() {
+ String nodeValues = " {" +
+ " 'node4':{" +
+ " 'node':'10.0.0.4:8987_solr'," +
+ " 'cores':1," +
+ " 'freedisk':884.7097854614258}," +
+ " 'node3':{" +
+ " 'node':'10.0.0.4:8989_solr'," +
+ " 'cores':1," +
+ " 'freedisk':884.7097854614258}," +
+ " 'node2':{" +
+ " 'node':'10.0.0.4:7574_solr'," +
+ " 'cores':1," +
+ " 'freedisk':884.7097854614258}," +
+ "}";
+
+ ClusterDataProvider provider = getClusterDataProvider((Map<String, Map>) Utils.fromJSONString(nodeValues), clusterState);
+ Map policies = (Map) Utils.fromJSONString("{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 50}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, 'nodeRole': 'overseer'}" +
+ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " ]" +
+ "}");
+ AutoScalingConfig config = new AutoScalingConfig(policies);
+ Policy policy = config.getPolicy();
+ Policy.Session session = policy.createSession(provider);
+ Policy.Suggester suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+
+ SolrRequest operation = suggester.getOperation();
+ assertNotNull(operation);
+ assertEquals("node2", operation.getParams().get("targetNode"));
+
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+ operation = suggester.getOperation();
+ assertNotNull(operation);
+ assertEquals("node3", operation.getParams().get("targetNode"));
+
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+ operation = suggester.getOperation();
+ assertNull(operation);
+
+ // lets change the policy such that all replicas that were on node1
+ // can now fit on node2
+ policies = (Map) Utils.fromJSONString("{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 50}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, 'nodeRole': 'overseer'}" +
+ " { 'replica': '<3', 'shard': '#EACH', 'node': '#ANY'}," +
+ " ]" +
+ "}");
+ config = new AutoScalingConfig(policies);
+ policy = config.getPolicy();
+ session = policy.createSession(provider);
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+
+ operation = suggester.getOperation();
+ assertNotNull(operation);
+ assertEquals("node2", operation.getParams().get("targetNode"));
+ assertEquals("r3", operation.getParams().get("replica"));
+
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+ operation = suggester.getOperation();
+ assertNotNull(operation);
+ assertEquals("node2", operation.getParams().get("targetNode"));
+ assertEquals("r5", operation.getParams().get("replica"));
+
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+ operation = suggester.getOperation();
+ assertEquals("node2", operation.getParams().get("targetNode"));
+ assertEquals("r1", operation.getParams().get("replica"));
+
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+ operation = suggester.getOperation();
+ assertNull(operation);
+
+ // now lets change the policy such that a node can have 2 shard2 replicas
+ policies = (Map) Utils.fromJSONString("{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 50}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, 'nodeRole': 'overseer'}" +
+ " { 'replica': '<2', 'shard': 'shard1', 'node': '#ANY'}," +
+ " { 'replica': '<3', 'shard': 'shard2', 'node': '#ANY'}," +
+ " ]" +
+ "}");
+ config = new AutoScalingConfig(policies);
+ policy = config.getPolicy();
+ session = policy.createSession(provider);
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+
+ operation = suggester.getOperation();
+ assertNotNull(operation);
+ assertEquals("node2", operation.getParams().get("targetNode"));
+ assertEquals("r3", operation.getParams().get("replica"));
+
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+ operation = suggester.getOperation();
+ assertNotNull(operation);
+ assertEquals("node2", operation.getParams().get("targetNode"));
+ assertEquals("r5", operation.getParams().get("replica"));
+
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.SRC_NODE, "node1");
+ operation = suggester.getOperation();
+ assertEquals("node3", operation.getParams().get("targetNode"));
+ assertEquals("r1", operation.getParams().get("replica"));
+ }
+
+ private static ClusterDataProvider dataProviderWithData(String data){
+ final Map m = (Map) Utils.fromJSONString(data);
+ Map replicaInfo = (Map) m.get("replicaInfo");
+ replicaInfo.forEach((node, val) -> {
+ Map m1 = (Map) val;
+ m1.forEach((coll, val2) -> {
+ Map m2 = (Map) val2;
+ m2.forEach((shard, val3) -> {
+ List l3 = (List) val3;
+ for (int i = 0; i < l3.size(); i++) {
+ Object o = l3.get(i);
+ Map m3 = (Map) o;
+ l3.set(i, new ReplicaInfo(m3.keySet().iterator().next().toString()
+ ,coll.toString(), shard.toString(), Replica.Type.get((String)m3.get("type")), new HashMap<>()));
+ }
+ });
+
+ });
+
+ });
+ return new ClusterDataProvider(){
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return (Map<String, Object>) Utils.getObjectByPath(m,false, Arrays.asList("nodeValues", node));
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return (Map<String, Map<String, List<ReplicaInfo>>>) Utils.getObjectByPath(m,false, Arrays.asList("replicaInfo", node));
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return (Collection<String>) m.get("liveNodes");
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+ };
+
+
+ }
+
+ public void testPolicyWithReplicaType() {
+ Map policies = (Map) Utils.fromJSONString("{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 50}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, 'nodeRole': 'overseer'}" +
+ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!ssd', type : TLOG }" +
+ " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!slowdisk' , type : PULL }" +
+ " ]" +
+ "}");
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "}");
+ Policy policy = new Policy(policies);
+ Policy.Suggester suggester = policy.createSession(getClusterDataProvider(nodeValues, clusterState))
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.REPLICATYPE, Replica.Type.PULL)
+ .hint(Hint.SHARD, "shard1");
+ SolrRequest op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals(Replica.Type.PULL.name(), op.getParams().get("type"));
+ assertEquals("PULL type node must be in 'slowdisk' node","node1", op.getParams().get("node"));
+
+ suggester = suggester.getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.REPLICATYPE, Replica.Type.PULL)
+ .hint(Hint.SHARD, "shard2");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals(Replica.Type.PULL.name(), op.getParams().get("type"));
+ assertEquals("PULL type node must be in 'slowdisk' node","node1", op.getParams().get("node"));
+
+ suggester = suggester.getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.REPLICATYPE, Replica.Type.TLOG)
+ .hint(Hint.SHARD, "shard1");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals(Replica.Type.TLOG.name(), op.getParams().get("type"));
+ assertEquals("TLOG type node must be in 'ssd' node","node3", op.getParams().get("node"));
+
+ suggester = suggester.getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.REPLICATYPE, Replica.Type.TLOG)
+ .hint(Hint.SHARD, "shard2");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals(Replica.Type.TLOG.name(), op.getParams().get("type"));
+ assertEquals("TLOG type node must be in 'ssd' node","node3", op.getParams().get("node"));
+
+ suggester = suggester.getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.REPLICATYPE, Replica.Type.TLOG)
+ .hint(Hint.SHARD, "shard2");
+ op = suggester.getOperation();
+ assertNull("No node should qualify for this" ,op);
+
+ }
+
public void testRow() {
- Row row = new Row("nodex", new Cell[]{new Cell(0, "node", "nodex")}, false, new HashMap<>(), new ArrayList<>());
- Row r1 = row.addReplica("c1", "s1");
- Row r2 = r1.addReplica("c1", "s1");
+ Row row = new Row("nodex", new Cell[]{new Cell(0, "node", "nodex")}, false, new HashMap<>(), new ArrayList<>(), true);
+ Row r1 = row.addReplica("c1", "s1", null);
+ Row r2 = r1.addReplica("c1", "s1",null);
assertEquals(1, r1.collectionVsShardVsReplicas.get("c1").get("s1").size());
assertEquals(2, r2.collectionVsShardVsReplicas.get("c1").get("s1").size());
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof Policy.ReplicaInfo);
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof Policy.ReplicaInfo);
+ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof ReplicaInfo);
+ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof ReplicaInfo);
}
public void testMerge() {
[18/18] lucene-solr:feature/autoscaling: SOLR-10931: Move
ReplicaCount and ReplicaInfo classes to new package. Fix compile and merge
issues.
Posted by sh...@apache.org.
SOLR-10931: Move ReplicaCount and ReplicaInfo classes to new package. Fix compile and merge issues.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/39c6fb2e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/39c6fb2e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/39c6fb2e
Branch: refs/heads/feature/autoscaling
Commit: 39c6fb2e3e7f178d90a7dd53b4e2afd3da9e4cc4
Parents: 8d66850
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Jun 28 12:21:29 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Jun 28 12:21:29 2017 +0530
----------------------------------------------------------------------
.../cloud/autoscaling/AutoScalingConfig.java | 1 +
.../cloud/autoscaling/ComputePlanAction.java | 1 +
.../autoscaling/AutoScalingHandlerTest.java | 1 +
.../client/solrj/cloud/autoscaling/Clause.java | 5 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 92 ++++++++++++++++++++
.../solrj/cloud/autoscaling/ReplicaInfo.java | 64 ++++++++++++++
.../client/solrj/cloud/autoscaling/Row.java | 1 -
.../solrj/impl/SolrClientDataProvider.java | 2 +-
.../solr/cloud/autoscaling/ReplicaCount.java | 92 --------------------
.../solr/cloud/autoscaling/ReplicaInfo.java | 64 --------------
.../solrj/cloud/autoscaling/TestPolicy.java | 1 +
11 files changed, 163 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingConfig.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingConfig.java
index fb812fe..5714ac9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingConfig.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingConfig.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.common.util.Utils;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 8e12a6c..8d9f8cd 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.common.cloud.ZkStateReader;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index b9577d1..06c2920 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index 8a5a121..7c82e6c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -31,7 +31,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
@@ -135,7 +134,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (tag != null && !params.contains(tag.name)) params.add(tag.name);
}
- public static class Condition {
+ class Condition {
final String name;
final Object val;
final Operand op;
@@ -181,7 +180,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
}
- static Condition parse(String s, Map m) {
+ Condition parse(String s, Map m) {
Object expectedVal = null;
Object val = m.get(s);
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
new file mode 100644
index 0000000..56a0205
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.Replica;
+
+class ReplicaCount extends Number implements MapWriter {
+ long nrt, tlog, pull;
+
+ public long total() {
+ return nrt + tlog + pull;
+ }
+
+ @Override
+ public int intValue() {
+ return (int) total();
+ }
+
+ @Override
+ public long longValue() {
+ return total();
+ }
+
+ @Override
+ public float floatValue() {
+ return total();
+ }
+
+ @Override
+ public double doubleValue() {
+ return total();
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(Replica.Type.NRT.name(), nrt);
+ ew.put(Replica.Type.PULL.name(), pull);
+ ew.put(Replica.Type.TLOG.name(), tlog);
+ ew.put("count", total());
+ }
+
+ public Long getVal(Replica.Type type) {
+ if (type == null) return total();
+ switch (type) {
+ case NRT:
+ return nrt;
+ case PULL:
+ return pull;
+ case TLOG:
+ return tlog;
+ }
+ return total();
+ }
+
+ public void increment(List<ReplicaInfo> infos) {
+ if (infos == null) return;
+ for (ReplicaInfo info : infos) {
+ switch (info.type) {
+ case NRT:
+ nrt++;
+ break;
+ case PULL:
+ pull++;
+ break;
+ case TLOG:
+ tlog++;
+ break;
+ default:
+ nrt++;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
new file mode 100644
index 0000000..f6b9db1
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.Replica;
+
+
+public class ReplicaInfo implements MapWriter {
+ final String name;
+ String core, collection, shard;
+ Replica.Type type;
+ Map<String, Object> variables;
+
+ public ReplicaInfo(String name, String coll, String shard, Replica.Type type, Map<String, Object> vals) {
+ this.name = name;
+ this.variables = vals;
+ this.collection = coll;
+ this.shard = shard;
+ this.type = type;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, (MapWriter) ew1 -> {
+ if (variables != null) {
+ for (Map.Entry<String, Object> e : variables.entrySet()) {
+ ew1.put(e.getKey(), e.getValue());
+ }
+ }
+ if (type != null) ew1.put("type", type.toString());
+ });
+ }
+
+ public String getCore() {
+ return core;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public String getShard() {
+ return shard;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 90559c1..c00249e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
index e6b7954..627de76 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
@@ -33,7 +33,7 @@ import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
import org.apache.solr.common.MapWriter;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java
deleted file mode 100644
index fed899e..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.cloud.Replica;
-
-class ReplicaCount extends Number implements MapWriter {
- long nrt, tlog, pull;
-
- public long total() {
- return nrt + tlog + pull;
- }
-
- @Override
- public int intValue() {
- return (int) total();
- }
-
- @Override
- public long longValue() {
- return total();
- }
-
- @Override
- public float floatValue() {
- return total();
- }
-
- @Override
- public double doubleValue() {
- return total();
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(Replica.Type.NRT.name(), nrt);
- ew.put(Replica.Type.PULL.name(), pull);
- ew.put(Replica.Type.TLOG.name(), tlog);
- ew.put("count", total());
- }
-
- public Long getVal(Replica.Type type) {
- if (type == null) return total();
- switch (type) {
- case NRT:
- return nrt;
- case PULL:
- return pull;
- case TLOG:
- return tlog;
- }
- return total();
- }
-
- public void increment(List<ReplicaInfo> infos) {
- if (infos == null) return;
- for (ReplicaInfo info : infos) {
- switch (info.type) {
- case NRT:
- nrt++;
- break;
- case PULL:
- pull++;
- break;
- case TLOG:
- tlog++;
- break;
- default:
- nrt++;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java
deleted file mode 100644
index 8d09997..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.cloud.Replica;
-
-
-public class ReplicaInfo implements MapWriter {
- final String name;
- String core, collection, shard;
- Replica.Type type;
- Map<String, Object> variables;
-
- public ReplicaInfo(String name, String coll, String shard, Replica.Type type, Map<String, Object> vals) {
- this.name = name;
- this.variables = vals;
- this.collection = coll;
- this.shard = shard;
- this.type = type;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(name, (MapWriter) ew1 -> {
- if (variables != null) {
- for (Map.Entry<String, Object> e : variables.entrySet()) {
- ew1.put(e.getKey(), e.getValue());
- }
- }
- if (type != null) ew1.put("type", type.toString());
- });
- }
-
- public String getCore() {
- return core;
- }
-
- public String getCollection() {
- return collection;
- }
-
- public String getShard() {
- return shard;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39c6fb2e/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 0fb3d05..a604a25 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -34,6 +34,7 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
+import org.apache.solr.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
[06/18] lucene-solr git commit: SOLR-10506: Fix memory leak (upon collection reload or ZooKeeper session expiry) in ZkIndexSchemaReader. (Torsten Bøgh Köster, Christine Poerschke, Jörg Rathlev, Mike Drob)
Posted by sh...@apache.org.
SOLR-10506: Fix memory leak (upon collection reload or ZooKeeper session expiry) in ZkIndexSchemaReader.
(Torsten Bøgh Köster, Christine Poerschke, Jörg Rathlev, Mike Drob)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/701c73d4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/701c73d4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/701c73d4
Branch: refs/heads/feature/autoscaling
Commit: 701c73d45a0188d006316e9a3168ad6b70e0fac9
Parents: e6f48ca
Author: Christine Poerschke <cp...@apache.org>
Authored: Tue Jun 27 14:04:58 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Tue Jun 27 15:07:40 2017 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../apache/solr/schema/ZkIndexSchemaReader.java | 105 ++++++++++++-------
.../apache/solr/schema/SchemaWatcherTest.java | 56 ++++++++++
3 files changed, 128 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/701c73d4/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7002993..3f25c26 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -211,6 +211,9 @@ Bug Fixes
* SOLR-10948: Fix extraction component to treat DatePointField the same as TrieDateField (hossman)
+* SOLR-10506: Fix memory leak (upon collection reload or ZooKeeper session expiry) in ZkIndexSchemaReader.
+ (Torsten Bøgh Köster, Christine Poerschke, Jörg Rathlev, Mike Drob)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/701c73d4/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java b/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java
index e719404..d83762a 100644
--- a/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java
+++ b/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java
@@ -42,7 +42,7 @@ public class ZkIndexSchemaReader implements OnReconnect {
private SolrZkClient zkClient;
private String managedSchemaPath;
private final String uniqueCoreId; // used in equals impl to uniquely identify the core that we're dependent on
- private boolean isRemoved = false;
+ private SchemaWatcher schemaWatcher;
public ZkIndexSchemaReader(ManagedIndexSchemaFactory managedIndexSchemaFactory, SolrCore solrCore) {
this.managedIndexSchemaFactory = managedIndexSchemaFactory;
@@ -58,16 +58,20 @@ public class ZkIndexSchemaReader implements OnReconnect {
CoreContainer cc = core.getCoreContainer();
if (cc.isZooKeeperAware()) {
log.debug("Removing ZkIndexSchemaReader OnReconnect listener as core "+core.getName()+" is shutting down.");
- ZkIndexSchemaReader.this.isRemoved = true;
cc.getZkController().removeOnReconnectListener(ZkIndexSchemaReader.this);
}
}
@Override
- public void postClose(SolrCore core) {}
+ public void postClose(SolrCore core) {
+ // The watcher is still registered with Zookeeper, and holds a
+ // reference to the schema reader, which indirectly references the
+ // SolrCore and would prevent it from being garbage collected.
+ schemaWatcher.discardReaderReference();
+ }
});
- createSchemaWatcher();
+ this.schemaWatcher = createSchemaWatcher();
zkLoader.getZkController().addOnReconnectListener(this);
}
@@ -76,39 +80,17 @@ public class ZkIndexSchemaReader implements OnReconnect {
return managedIndexSchemaFactory.getSchemaUpdateLock();
}
- public void createSchemaWatcher() {
+ /**
+ * Creates a schema watcher and returns it for controlling purposes.
+ *
+ * @return the registered {@linkplain SchemaWatcher}.
+ */
+ public SchemaWatcher createSchemaWatcher() {
log.info("Creating ZooKeeper watch for the managed schema at " + managedSchemaPath);
+ SchemaWatcher watcher = new SchemaWatcher(this);
try {
- zkClient.exists(managedSchemaPath, new Watcher() {
- @Override
- public void process(WatchedEvent event) {
-
- if (ZkIndexSchemaReader.this.isRemoved) {
- return; // the core for this reader has already been removed, don't process this event
- }
-
- // session events are not change events, and do not remove the watcher
- if (Event.EventType.None.equals(event.getType())) {
- return;
- }
- log.info("A schema change: {}, has occurred - updating schema from ZooKeeper ...", event);
- try {
- updateSchema(this, -1);
- } catch (KeeperException e) {
- if (e.code() == KeeperException.Code.SESSIONEXPIRED || e.code() == KeeperException.Code.CONNECTIONLOSS) {
- log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
- return;
- }
- log.error("", e);
- throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "", e);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- log.warn("", e);
- }
- }
- }, true);
+ zkClient.exists(managedSchemaPath, watcher, true);
} catch (KeeperException e) {
final String msg = "Error creating ZooKeeper watch for the managed schema";
log.error(msg, e);
@@ -118,6 +100,56 @@ public class ZkIndexSchemaReader implements OnReconnect {
Thread.currentThread().interrupt();
log.warn("", e);
}
+
+ return watcher;
+ }
+
+ /**
+ * Watches for schema changes and triggers updates in the {@linkplain ZkIndexSchemaReader}.
+ */
+ public static class SchemaWatcher implements Watcher {
+
+ private ZkIndexSchemaReader schemaReader;
+
+ public SchemaWatcher(ZkIndexSchemaReader reader) {
+ this.schemaReader = reader;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ ZkIndexSchemaReader indexSchemaReader = schemaReader;
+
+ if (indexSchemaReader == null) {
+ return; // the core for this reader has already been removed, don't process this event
+ }
+
+ // session events are not change events, and do not remove the watcher
+ if (Event.EventType.None.equals(event.getType())) {
+ return;
+ }
+ log.info("A schema change: {}, has occurred - updating schema from ZooKeeper ...", event);
+ try {
+ indexSchemaReader.updateSchema(this, -1);
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+ return;
+ }
+ log.error("", e);
+ throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ }
+ }
+
+ /**
+ * Discard the reference to the {@code ZkIndexSchemaReader}.
+ */
+ public void discardReaderReference() {
+ schemaReader = null;
+ }
}
public ManagedIndexSchema refreshSchemaFromZk(int expectedZkVersion) throws KeeperException, InterruptedException {
@@ -125,7 +157,8 @@ public class ZkIndexSchemaReader implements OnReconnect {
return managedIndexSchemaFactory.getSchema();
}
- private void updateSchema(Watcher watcher, int expectedZkVersion) throws KeeperException, InterruptedException {
+ // package visibility for test purposes
+ void updateSchema(Watcher watcher, int expectedZkVersion) throws KeeperException, InterruptedException {
Stat stat = new Stat();
synchronized (getSchemaUpdateLock()) {
final ManagedIndexSchema oldSchema = managedIndexSchemaFactory.getSchema();
@@ -157,7 +190,7 @@ public class ZkIndexSchemaReader implements OnReconnect {
public void command() {
try {
// setup a new watcher to get notified when the managed schema changes
- createSchemaWatcher();
+ schemaWatcher = createSchemaWatcher();
// force update now as the schema may have changed while our zk session was expired
updateSchema(null, -1);
} catch (Exception exc) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/701c73d4/solr/core/src/test/org/apache/solr/schema/SchemaWatcherTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/SchemaWatcherTest.java b/solr/core/src/test/org/apache/solr/schema/SchemaWatcherTest.java
new file mode 100644
index 0000000..4d46aad
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/schema/SchemaWatcherTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.schema;
+
+import org.apache.solr.schema.ZkIndexSchemaReader.SchemaWatcher;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
+public class SchemaWatcherTest {
+
+ private ZkIndexSchemaReader mockSchemaReader;
+ private SchemaWatcher schemaWatcher;
+
+ @Before
+ public void setUp() throws Exception {
+ mockSchemaReader = mock(ZkIndexSchemaReader.class);
+ schemaWatcher = new SchemaWatcher(mockSchemaReader);
+ }
+
+ @Test
+ public void testProcess() throws Exception {
+ schemaWatcher.process(new WatchedEvent(EventType.NodeDataChanged, KeeperState.SyncConnected, "/test"));
+ verify(mockSchemaReader).updateSchema(schemaWatcher, -1);
+ }
+
+ @Test
+ public void testDiscardReaderReference() throws Exception {
+ schemaWatcher.discardReaderReference();
+
+ schemaWatcher.process(new WatchedEvent(EventType.NodeDataChanged, KeeperState.SyncConnected, "/test"));
+ // after discardReaderReference, SchemaWatcher should no longer hold a ref to the reader
+ verifyZeroInteractions(mockSchemaReader);
+ }
+}
[08/18] lucene-solr:feature/autoscaling: LUCENE-7737: Remove
spatial-extras dependency on queries module
Posted by sh...@apache.org.
LUCENE-7737: Remove spatial-extras dependency on queries module
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2f2e00ff
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2f2e00ff
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2f2e00ff
Branch: refs/heads/feature/autoscaling
Commit: 2f2e00ffe235ba590f43bf07dabd4201e6da53c6
Parents: 1a278ae
Author: Alan Woodward <ro...@apache.org>
Authored: Tue Jun 13 09:41:23 2017 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Tue Jun 27 15:53:39 2017 +0100
----------------------------------------------------------------------
.../lucene/spatial-extras/spatial-extras.iml | 5 +-
lucene/CHANGES.txt | 5 +
.../byTask/feeds/SpatialFileQueryMaker.java | 15 +--
.../org/apache/lucene/search/DoubleValues.java | 21 ++++
.../lucene/search/DoubleValuesSource.java | 7 +-
.../lucene/queries/function/ValueSource.java | 63 +++++++++++
lucene/spatial-extras/build.xml | 8 +-
.../org/apache/lucene/spatial/ShapeValues.java | 41 +++++++
.../lucene/spatial/ShapeValuesSource.java | 34 ++++++
.../apache/lucene/spatial/SpatialStrategy.java | 19 ++--
.../bbox/BBoxOverlapRatioValueSource.java | 7 +-
.../spatial/bbox/BBoxSimilarityValueSource.java | 79 +++++++------
.../lucene/spatial/bbox/BBoxStrategy.java | 13 +--
.../lucene/spatial/bbox/BBoxValueSource.java | 74 +++---------
.../composite/CompositeSpatialStrategy.java | 14 +--
.../spatial/composite/CompositeVerifyQuery.java | 32 ++----
.../composite/IntersectsRPTVerifyQuery.java | 21 ++--
.../prefix/NumberRangePrefixTreeStrategy.java | 4 +-
.../spatial/prefix/PrefixTreeStrategy.java | 4 +-
.../serialized/SerializedDVStrategy.java | 110 ++++--------------
.../spatial/util/CachingDoubleValueSource.java | 61 +++++-----
.../util/DistanceToShapeValueSource.java | 68 +++++------
.../util/ReciprocalDoubleValuesSource.java | 96 ++++++++++++++++
.../spatial/util/ShapeAreaValueSource.java | 67 ++++-------
.../ShapeFieldCacheDistanceValueSource.java | 59 +++++-----
.../spatial/util/ShapePredicateValueSource.java | 113 -------------------
.../spatial/util/ShapeValuesPredicate.java | 99 ++++++++++++++++
.../spatial/vector/DistanceValueSource.java | 72 +++++-------
.../spatial/vector/PointVectorStrategy.java | 95 ++++++++++++++--
.../lucene/spatial/DistanceStrategyTest.java | 6 -
.../apache/lucene/spatial/SpatialExample.java | 12 +-
.../apache/lucene/spatial/StrategyTestCase.java | 39 +++----
.../lucene/spatial/spatial4j/Geo3dRptTest.java | 2 +-
.../Geo3dShapeRectRelationTestCase.java | 2 +-
.../org/apache/solr/legacy/BBoxStrategy.java | 13 +--
.../org/apache/solr/legacy/BBoxValueSource.java | 82 ++++----------
.../apache/solr/legacy/DistanceValueSource.java | 81 +++++--------
.../apache/solr/legacy/PointVectorStrategy.java | 17 ++-
.../transform/GeoTransformerFactory.java | 33 +++---
.../solr/schema/AbstractSpatialFieldType.java | 10 +-
.../java/org/apache/solr/schema/BBoxField.java | 8 +-
.../solr/schema/LatLonPointSpatialField.java | 46 ++++----
.../schema/RptWithGeometrySpatialField.java | 60 ++++------
.../distance/GeoDistValueSourceParser.java | 8 +-
.../apache/solr/search/TestSolr4Spatial.java | 10 +-
45 files changed, 886 insertions(+), 849 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml
----------------------------------------------------------------------
diff --git a/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml b/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml
index 6285d26..8e9d887 100644
--- a/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml
+++ b/dev-tools/idea/lucene/spatial-extras/spatial-extras.iml
@@ -24,10 +24,7 @@
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" scope="TEST" module-name="lucene-test-framework" />
<orderEntry type="module" module-name="lucene-core" />
- <orderEntry type="module" module-name="queries" />
- <orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="spatial3d" />
- <orderEntry type="module" module-name="backward-codecs" />
<orderEntry type="module" module-name="analysis-common" scope="TEST"/>
</component>
-</module>
\ No newline at end of file
+</module>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 517d937..69ba53f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -92,6 +92,11 @@ API Changes
* LUCENE-7723: DoubleValuesSource enforces implementation of equals() and
hashCode() (Alan Woodward)
+* LUCENE-7737: The spatial-extras module no longer has a dependency on the
+ queries module. All uses of ValueSource are either replaced with core
+ DoubleValuesSource extensions, or with the new ShapeValuesSource and
+ ShapeValuesPredicate classes (Alan Woodward, David Smiley)
+
Bug Fixes
* LUCENE-7626: IndexWriter will no longer accept broken token offsets
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java
----------------------------------------------------------------------
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java
index b6b8f50..28bd821 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SpatialFileQueryMaker.java
@@ -21,16 +21,14 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
-import org.locationtech.spatial4j.shape.Shape;
import org.apache.lucene.benchmark.byTask.utils.Config;
-import org.apache.lucene.queries.function.FunctionQuery;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.queries.function.FunctionScoreQuery;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Query;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
+import org.locationtech.spatial4j.shape.Shape;
/**
* Reads spatial data from the body field docs from an internally created {@link LineDocSource}.
@@ -102,11 +100,8 @@ public class SpatialFileQueryMaker extends AbstractQueryMaker {
Query filterQuery = strategy.makeQuery(args);
if (score) {
//wrap with distance computing query
- ValueSource valueSource = strategy.makeDistanceValueSource(shape.getCenter());
- return new BooleanQuery.Builder()
- .add(new FunctionQuery(valueSource), BooleanClause.Occur.MUST)//matches everything and provides score
- .add(filterQuery, BooleanClause.Occur.FILTER)//filters (score isn't used)
- .build();
+ DoubleValuesSource valueSource = strategy.makeDistanceValueSource(shape.getCenter());
+ return new FunctionScoreQuery(filterQuery, valueSource);
} else {
return filterQuery; // assume constant scoring
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java
index 4f12390..84167bc 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java
@@ -35,4 +35,25 @@ public abstract class DoubleValues {
*/
public abstract boolean advanceExact(int doc) throws IOException;
+ /**
+ * Wrap a DoubleValues instance, returning a default if the wrapped instance has no value
+ */
+ public static DoubleValues withDefault(DoubleValues in, double missingValue) {
+ return new DoubleValues() {
+
+ boolean hasValue = false;
+
+ @Override
+ public double doubleValue() throws IOException {
+ return hasValue ? in.doubleValue() : missingValue;
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ hasValue = in.advanceExact(doc);
+ return true;
+ }
+ };
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
index 342c08b..84e5eb6 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
@@ -64,7 +64,12 @@ public abstract class DoubleValuesSource {
* @return an Explanation for the value
* @throws IOException if an {@link IOException} occurs
*/
- public abstract Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException;
+ public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
+ DoubleValues dv = getValues(ctx, DoubleValuesSource.constant(scoreExplanation.getValue()).getValues(ctx, null));
+ if (dv.advanceExact(docId))
+ return Explanation.match((float) dv.doubleValue(), this.toString());
+ return Explanation.noMatch(this.toString());
+ }
/**
* Create a sort field based on the value of this producer
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
index cb3faa3..5f38226 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
@@ -258,6 +258,69 @@ public abstract class ValueSource {
}
+ public static ValueSource fromDoubleValuesSource(DoubleValuesSource in) {
+ return new FromDoubleValuesSource(in);
+ }
+
+ private static class FromDoubleValuesSource extends ValueSource {
+
+ final DoubleValuesSource in;
+
+ private FromDoubleValuesSource(DoubleValuesSource in) {
+ this.in = in;
+ }
+
+ @Override
+ public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ Scorer scorer = (Scorer) context.get("scorer");
+ DoubleValues scores = scorer == null ? null : DoubleValuesSource.fromScorer(scorer);
+ DoubleValues inner = in.getValues(readerContext, scores);
+ return new FunctionValues() {
+ @Override
+ public String toString(int doc) throws IOException {
+ return in.toString();
+ }
+
+ @Override
+ public float floatVal(int doc) throws IOException {
+ if (inner.advanceExact(doc) == false)
+ return 0;
+ return (float) inner.doubleValue();
+ }
+
+ @Override
+ public double doubleVal(int doc) throws IOException {
+ if (inner.advanceExact(doc) == false)
+ return 0;
+ return inner.doubleValue();
+ }
+
+ @Override
+ public boolean exists(int doc) throws IOException {
+ return inner.advanceExact(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ FromDoubleValuesSource that = (FromDoubleValuesSource) o;
+ return Objects.equals(in, that.in);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(in);
+ }
+
+ @Override
+ public String description() {
+ return in.toString();
+ }
+ }
+
//
// Sorting by function
//
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/build.xml
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/build.xml b/lucene/spatial-extras/build.xml
index e9cc29c..4cfa4e5 100644
--- a/lucene/spatial-extras/build.xml
+++ b/lucene/spatial-extras/build.xml
@@ -31,7 +31,6 @@
<path id="classpath">
<path refid="base.classpath"/>
<path refid="spatialjar"/>
- <pathelement path="${queries.jar}" />
<pathelement path="${spatial3d.jar}" />
</path>
@@ -41,15 +40,12 @@
<pathelement path="src/test-files" />
</path>
- <target name="compile-core" depends="jar-backward-codecs,jar-queries,jar-misc,jar-spatial3d,common.compile-core" />
+ <target name="compile-core" depends="jar-spatial3d,common.compile-core" />
- <target name="javadocs" depends="javadocs-backward-codecs,javadocs-queries,javadocs-misc,javadocs-spatial3d,compile-core,check-javadocs-uptodate"
+ <target name="javadocs" depends="javadocs-spatial3d,compile-core,check-javadocs-uptodate"
unless="javadocs-uptodate-${name}">
<invoke-module-javadoc>
<links>
- <link href="../backward-codecs"/>
- <link href="../queries"/>
- <link href="../misc"/>
<link href="../spatial3d"/>
</links>
</invoke-module-javadoc>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValues.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValues.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValues.java
new file mode 100644
index 0000000..51323c9
--- /dev/null
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValues.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.spatial;
+
+import java.io.IOException;
+
+import org.locationtech.spatial4j.shape.Shape;
+
+/**
+ * Iterator over {@link Shape} objects for an index segment
+ */
+public abstract class ShapeValues {
+
+ /**
+ * Advance the iterator to the given document
+ * @param doc the document to advance to
+ * @return {@code true} if there is a value for this document
+ */
+ public abstract boolean advanceExact(int doc) throws IOException;
+
+ /**
+ * Returns a {@link Shape} for the current document
+ */
+ public abstract Shape value() throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValuesSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValuesSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValuesSource.java
new file mode 100644
index 0000000..b96edd8
--- /dev/null
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/ShapeValuesSource.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.spatial;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+/**
+ * Produces {@link ShapeValues} per-segment
+ */
+public abstract class ShapeValuesSource {
+
+ /**
+ * Get a {@link ShapeValues} instance for the given leaf reader context
+ */
+ public abstract ShapeValues getValues(LeafReaderContext ctx) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java
index d980ba9..215fa1d 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/SpatialStrategy.java
@@ -16,15 +16,15 @@
*/
package org.apache.lucene.spatial;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.spatial.query.SpatialArgs;
+import org.apache.lucene.spatial.util.ReciprocalDoubleValuesSource;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.shape.Point;
import org.locationtech.spatial4j.shape.Rectangle;
import org.locationtech.spatial4j.shape.Shape;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.valuesource.ReciprocalFloatFunction;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.spatial.query.SpatialArgs;
/**
* The SpatialStrategy encapsulates an approach to indexing and searching based
@@ -103,7 +103,7 @@ public abstract class SpatialStrategy {
* See {@link #makeDistanceValueSource(org.locationtech.spatial4j.shape.Point, double)} called with
* a multiplier of 1.0 (i.e. units of degrees).
*/
- public ValueSource makeDistanceValueSource(Point queryPoint) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint) {
return makeDistanceValueSource(queryPoint, 1.0);
}
@@ -113,7 +113,7 @@ public abstract class SpatialStrategy {
* then the closest one is chosen. The result is multiplied by {@code multiplier}, which
* conveniently is used to get the desired units.
*/
- public abstract ValueSource makeDistanceValueSource(Point queryPoint, double multiplier);
+ public abstract DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier);
/**
* Make a Query based principally on {@link org.apache.lucene.spatial.query.SpatialOperation}
@@ -133,13 +133,14 @@ public abstract class SpatialStrategy {
* scores will be 1 for indexed points at the center of the query shape and as
* low as ~0.1 at its furthest edges.
*/
- public final ValueSource makeRecipDistanceValueSource(Shape queryShape) {
+ public final DoubleValuesSource makeRecipDistanceValueSource(Shape queryShape) {
Rectangle bbox = queryShape.getBoundingBox();
double diagonalDist = ctx.getDistCalc().distance(
ctx.makePoint(bbox.getMinX(), bbox.getMinY()), bbox.getMaxX(), bbox.getMaxY());
double distToEdge = diagonalDist * 0.5;
float c = (float)distToEdge * 0.1f;//one tenth
- return new ReciprocalFloatFunction(makeDistanceValueSource(queryShape.getCenter(), 1.0), 1f, c, c);
+ DoubleValuesSource distance = makeDistanceValueSource(queryShape.getCenter(), 1.0);
+ return new ReciprocalDoubleValuesSource(c, distance);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java
index 101f373..e83279e 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.java
@@ -18,9 +18,8 @@ package org.apache.lucene.spatial.bbox;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.Explanation;
-
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.locationtech.spatial4j.shape.Rectangle;
/**
@@ -79,7 +78,7 @@ public class BBoxOverlapRatioValueSource extends BBoxSimilarityValueSource {
* @param queryTargetProportion see class javadocs. Between 0 and 1.
* @param minSideLength see class javadocs. 0.0 will effectively disable.
*/
- public BBoxOverlapRatioValueSource(ValueSource rectValueSource, boolean isGeo, Rectangle queryExtent,
+ public BBoxOverlapRatioValueSource(ShapeValuesSource rectValueSource, boolean isGeo, Rectangle queryExtent,
double queryTargetProportion, double minSideLength) {
super(rectValueSource);
this.isGeo = isGeo;
@@ -94,7 +93,7 @@ public class BBoxOverlapRatioValueSource extends BBoxSimilarityValueSource {
/** Construct with 75% weighting towards target (roughly GeoPortal's default), geo degrees assumed, no
* minimum side length. */
- public BBoxOverlapRatioValueSource(ValueSource rectValueSource, Rectangle queryExtent) {
+ public BBoxOverlapRatioValueSource(ShapeValuesSource rectValueSource, Rectangle queryExtent) {
this(rectValueSource, true, queryExtent, 0.25, 0.0);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
index b1424e0..aed9f6c 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
@@ -17,78 +17,58 @@
package org.apache.lucene.spatial.bbox;
import java.io.IOException;
-import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
-
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.locationtech.spatial4j.shape.Rectangle;
/**
* A base class for calculating a spatial relevance rank per document from a provided
- * {@link ValueSource} in which {@link FunctionValues#objectVal(int)} returns a {@link
- * org.locationtech.spatial4j.shape.Rectangle}.
+ * {@link ShapeValuesSource} returning a {@link
+ * org.locationtech.spatial4j.shape.Rectangle} per-document.
* <p>
* Implementers: remember to implement equals and hashCode if you have
* fields!
*
* @lucene.experimental
*/
-public abstract class BBoxSimilarityValueSource extends ValueSource {
+public abstract class BBoxSimilarityValueSource extends DoubleValuesSource {
- private final ValueSource bboxValueSource;
+ private final ShapeValuesSource bboxValueSource;
- public BBoxSimilarityValueSource(ValueSource bboxValueSource) {
+ public BBoxSimilarityValueSource(ShapeValuesSource bboxValueSource) {
this.bboxValueSource = bboxValueSource;
}
@Override
- public void createWeight(Map context, IndexSearcher searcher) throws IOException {
- bboxValueSource.createWeight(context, searcher);
- }
-
- @Override
- public String description() {
- return getClass().getSimpleName()+"(" + bboxValueSource.description() + "," + similarityDescription() + ")";
+ public String toString() {
+ return getClass().getSimpleName()+"(" + bboxValueSource.toString() + "," + similarityDescription() + ")";
}
- /** A comma-separated list of configurable items of the subclass to put into {@link #description()}. */
+ /** A comma-separated list of configurable items of the subclass to put into {@link #toString()}. */
protected abstract String similarityDescription();
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
- final FunctionValues shapeValues = bboxValueSource.getValues(context, readerContext);
-
- return new DoubleDocValues(this) {
+ final ShapeValues shapeValues = bboxValueSource.getValues(readerContext);
+ return DoubleValues.withDefault(new DoubleValues() {
@Override
- public double doubleVal(int doc) throws IOException {
- //? limit to Rect or call getBoundingBox()? latter would encourage bad practice
- final Rectangle rect = (Rectangle) shapeValues.objectVal(doc);
- return rect==null ? 0 : score(rect, null);
+ public double doubleValue() throws IOException {
+ return score((Rectangle) shapeValues.value(), null);
}
@Override
- public boolean exists(int doc) throws IOException {
- return shapeValues.exists(doc);
+ public boolean advanceExact(int doc) throws IOException {
+ return shapeValues.advanceExact(doc);
}
+ }, 0);
- @Override
- public Explanation explain(int doc) throws IOException {
- final Rectangle rect = (Rectangle) shapeValues.objectVal(doc);
- if (rect == null) {
- return Explanation.noMatch("no rect");
- }
- AtomicReference<Explanation> explanation = new AtomicReference<>();
- score(rect, explanation);
- return explanation.get();
- }
- };
}
/**
@@ -115,4 +95,23 @@ public abstract class BBoxSimilarityValueSource extends ValueSource {
public int hashCode() {
return bboxValueSource.hashCode();
}
+
+ @Override
+ public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
+ DoubleValues dv = getValues(ctx, DoubleValuesSource.constant(scoreExplanation.getValue()).getValues(ctx, null));
+ if (dv.advanceExact(docId)) {
+ AtomicReference<Explanation> explanation = new AtomicReference<>();
+ final ShapeValues shapeValues = bboxValueSource.getValues(ctx);
+ if (shapeValues.advanceExact(docId)) {
+ score((Rectangle) shapeValues.value(), explanation);
+ return explanation.get();
+ }
+ }
+ return Explanation.noMatch(this.toString());
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
index 7536b60..5029d0e 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
@@ -25,12 +25,13 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
@@ -211,23 +212,21 @@ public class BBoxStrategy extends SpatialStrategy {
//---------------------------------
/**
- * Provides access to each rectangle per document as a ValueSource in which
- * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)} returns a {@link
- * Shape}.
+ * Provides access to each rectangle per document as a {@link ShapeValuesSource}
*/ //TODO raise to SpatialStrategy
- public ValueSource makeShapeValueSource() {
+ public ShapeValuesSource makeShapeValueSource() {
return new BBoxValueSource(this);
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
//TODO if makeShapeValueSource gets lifted to the top; this could become a generic impl.
return new DistanceToShapeValueSource(makeShapeValueSource(), queryPoint, multiplier, ctx);
}
/** Returns a similarity based on {@link BBoxOverlapRatioValueSource}. This is just a
* convenience method. */
- public ValueSource makeOverlapRatioValueSource(Rectangle queryBox, double queryTargetProportion) {
+ public DoubleValuesSource makeOverlapRatioValueSource(Rectangle queryBox, double queryTargetProportion) {
return new BBoxOverlapRatioValueSource(
makeShapeValueSource(), ctx.isGeo(), queryBox, queryTargetProportion, 0.0);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
index ef9f54e..6f7fc9f 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/bbox/BBoxValueSource.java
@@ -17,24 +17,22 @@
package org.apache.lucene.spatial.bbox;
import java.io.IOException;
-import java.util.Map;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.Explanation;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
/**
- * A ValueSource in which the indexed Rectangle is returned from
- * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)}.
+ * A ShapeValuesSource returning a Rectangle from each document derived from four numeric fields
*
* @lucene.internal
*/
-class BBoxValueSource extends ValueSource {
+class BBoxValueSource extends ShapeValuesSource {
private final BBoxStrategy strategy;
@@ -43,12 +41,12 @@ class BBoxValueSource extends ValueSource {
}
@Override
- public String description() {
+ public String toString() {
return "bboxShape(" + strategy.getFieldName() + ")";
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public ShapeValues getValues(LeafReaderContext readerContext) throws IOException {
LeafReader reader = readerContext.reader();
final NumericDocValues minX = DocValues.getNumeric(reader, strategy.field_minX);
final NumericDocValues minY = DocValues.getNumeric(reader, strategy.field_minY);
@@ -58,61 +56,23 @@ class BBoxValueSource extends ValueSource {
//reused
final Rectangle rect = strategy.getSpatialContext().makeRectangle(0,0,0,0);
- return new FunctionValues() {
- private int lastDocID = -1;
-
- private double getDocValue(NumericDocValues values, int doc) throws IOException {
- int curDocID = values.docID();
- if (doc > curDocID) {
- curDocID = values.advance(doc);
- }
- if (doc == curDocID) {
- return Double.longBitsToDouble(values.longValue());
- } else {
- return 0.0;
- }
- }
+ return new ShapeValues() {
@Override
- public Object objectVal(int doc) throws IOException {
- if (doc < lastDocID) {
- throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + doc);
- }
- lastDocID = doc;
-
- double minXValue = getDocValue(minX, doc);
- if (minX.docID() != doc) {
- return null;
- } else {
- double minYValue = getDocValue(minY, doc);
- double maxXValue = getDocValue(maxX, doc);
- double maxYValue = getDocValue(maxY, doc);
- rect.reset(minXValue, maxXValue, minYValue, maxYValue);
- return rect;
- }
+ public boolean advanceExact(int doc) throws IOException {
+ return minX.advanceExact(doc) && minY.advanceExact(doc) && maxX.advanceExact(doc) && maxY.advanceExact(doc);
}
@Override
- public String strVal(int doc) throws IOException {//TODO support WKT output once Spatial4j does
- Object v = objectVal(doc);
- return v == null ? null : v.toString();
+ public Shape value() throws IOException {
+ double minXValue = Double.longBitsToDouble(minX.longValue());
+ double minYValue = Double.longBitsToDouble(minY.longValue());
+ double maxXValue = Double.longBitsToDouble(maxX.longValue());
+ double maxYValue = Double.longBitsToDouble(maxY.longValue());
+ rect.reset(minXValue, maxXValue, minYValue, maxYValue);
+ return rect;
}
- @Override
- public boolean exists(int doc) throws IOException {
- getDocValue(minX, doc);
- return minX.docID() == doc;
- }
-
- @Override
- public Explanation explain(int doc) throws IOException {
- return Explanation.match(Float.NaN, toString(doc));
- }
-
- @Override
- public String toString(int doc) throws IOException {
- return description() + '=' + strVal(doc);
- }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java
index de5bb61..348b7d6 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeSpatialStrategy.java
@@ -20,10 +20,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Shape;
import org.apache.lucene.document.Field;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Query;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
@@ -32,7 +30,9 @@ import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
import org.apache.lucene.spatial.query.UnsupportedSpatialOperation;
import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
-import org.apache.lucene.spatial.util.ShapePredicateValueSource;
+import org.apache.lucene.spatial.util.ShapeValuesPredicate;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
/**
* A composite {@link SpatialStrategy} based on {@link RecursivePrefixTreeStrategy} (RPT) and
@@ -86,7 +86,7 @@ public class CompositeSpatialStrategy extends SpatialStrategy {
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
//TODO consider indexing center-point in DV? Guarantee contained by the shape, which could then be used for
// other purposes like faster WITHIN predicate?
throw new UnsupportedOperationException();
@@ -108,8 +108,8 @@ public class CompositeSpatialStrategy extends SpatialStrategy {
throw new UnsupportedSpatialOperation(pred);
}
- final ShapePredicateValueSource predicateValueSource =
- new ShapePredicateValueSource(geometryStrategy.makeShapeValueSource(), pred, args.getShape());
+ final ShapeValuesPredicate predicateValueSource =
+ new ShapeValuesPredicate(geometryStrategy.makeShapeValueSource(), pred, args.getShape());
//System.out.println("PredOpt: " + optimizePredicates);
if (pred == SpatialOperation.Intersects && optimizePredicates) {
// We have a smart Intersects impl
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java
index d556efa..36a9eff 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/CompositeVerifyQuery.java
@@ -17,12 +17,9 @@
package org.apache.lucene.spatial.composite;
import java.io.IOException;
-import java.util.Map;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.IndexSearcher;
@@ -30,19 +27,20 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
+import org.apache.lucene.spatial.util.ShapeValuesPredicate;
/**
* A Query that considers an "indexQuery" to have approximate results, and a follow-on
- * {@link ValueSource}/{@link FunctionValues#boolVal(int)} is called to verify each hit
- * from {@link TwoPhaseIterator#matches()}.
+ * ShapeValuesSource is called to verify each hit from {@link TwoPhaseIterator#matches()}.
*
* @lucene.experimental
*/
public class CompositeVerifyQuery extends Query {
- final Query indexQuery;//approximation (matches more than needed)
- final ValueSource predicateValueSource;//we call boolVal(doc)
- public CompositeVerifyQuery(Query indexQuery, ValueSource predicateValueSource) {
+ private final Query indexQuery;//approximation (matches more than needed)
+ private final ShapeValuesPredicate predicateValueSource;
+
+ public CompositeVerifyQuery(Query indexQuery, ShapeValuesPredicate predicateValueSource) {
this.indexQuery = indexQuery;
this.predicateValueSource = predicateValueSource;
}
@@ -84,7 +82,6 @@ public class CompositeVerifyQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
final Weight indexQueryWeight = indexQuery.createWeight(searcher, false, boost);//scores aren't unsupported
- final Map valueSourceContext = ValueSource.newContext(searcher);
return new ConstantScoreWeight(this, boost) {
@@ -96,21 +93,8 @@ public class CompositeVerifyQuery extends Query {
return null;
}
- final FunctionValues predFuncValues = predicateValueSource.getValues(valueSourceContext, context);
-
- final TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(indexQueryScorer.iterator()) {
- @Override
- public boolean matches() throws IOException {
- return predFuncValues.boolVal(indexQueryScorer.docID());
- }
-
- @Override
- public float matchCost() {
- return 100; // TODO: use cost of predFuncValues.boolVal()
- }
- };
-
- return new ConstantScoreScorer(this, score(), twoPhaseIterator);
+ final TwoPhaseIterator predFuncValues = predicateValueSource.iterator(context, indexQueryScorer.iterator());
+ return new ConstantScoreScorer(this, score(), predFuncValues);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java
index ce8c207..a6ea3a3 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/composite/IntersectsRPTVerifyQuery.java
@@ -17,11 +17,8 @@
package org.apache.lucene.spatial.composite;
import java.io.IOException;
-import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSet;
@@ -34,6 +31,7 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.spatial.prefix.AbstractVisitingPrefixTreeQuery;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
+import org.apache.lucene.spatial.util.ShapeValuesPredicate;
import org.apache.lucene.util.DocIdSetBuilder;
import org.locationtech.spatial4j.shape.Shape;
import org.locationtech.spatial4j.shape.SpatialRelation;
@@ -41,17 +39,17 @@ import org.locationtech.spatial4j.shape.SpatialRelation;
/**
* A spatial Intersects predicate that distinguishes an approximated match from an exact match based on which cells
* are within the query shape. It exposes a {@link TwoPhaseIterator} that will verify a match with a provided
- * predicate in the form of a {@link ValueSource} by calling {@link FunctionValues#boolVal(int)}.
+ * predicate in the form of an ShapeValuesPredicate.
*
* @lucene.internal
*/
public class IntersectsRPTVerifyQuery extends Query {
private final IntersectsDifferentiatingQuery intersectsDiffQuery;
- private final ValueSource predicateValueSource; // we call FunctionValues.boolVal(doc)
+ private final ShapeValuesPredicate predicateValueSource;
public IntersectsRPTVerifyQuery(Shape queryShape, String fieldName, SpatialPrefixTree grid, int detailLevel,
- int prefixGridScanLevel, ValueSource predicateValueSource) {
+ int prefixGridScanLevel, ShapeValuesPredicate predicateValueSource) {
this.predicateValueSource = predicateValueSource;
this.intersectsDiffQuery = new IntersectsDifferentiatingQuery(queryShape, fieldName, grid, detailLevel,
prefixGridScanLevel);
@@ -83,7 +81,6 @@ public class IntersectsRPTVerifyQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
- final Map valueSourceContext = ValueSource.newContext(searcher);
return new ConstantScoreWeight(this, boost) {
@Override
@@ -110,9 +107,10 @@ public class IntersectsRPTVerifyQuery extends Query {
exactIterator = null;
}
- final FunctionValues predFuncValues = predicateValueSource.getValues(valueSourceContext, context);
-
final TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(approxDISI) {
+
+ final TwoPhaseIterator predFuncValues = predicateValueSource.iterator(context, approxDISI);
+
@Override
public boolean matches() throws IOException {
final int doc = approxDISI.docID();
@@ -124,13 +122,12 @@ public class IntersectsRPTVerifyQuery extends Query {
return true;
}
}
-
- return predFuncValues.boolVal(doc);
+ return predFuncValues.matches();
}
@Override
public float matchCost() {
- return 100; // TODO: use cost of exactIterator.advance() and predFuncValues.boolVal()
+ return 100; // TODO: use cost of exactIterator.advance() and predFuncValues.cost()
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java
index 8367644..d720215 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/NumberRangePrefixTreeStrategy.java
@@ -23,7 +23,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.NumberRangePrefixTree;
import org.apache.lucene.util.Bits;
@@ -76,7 +76,7 @@ public class NumberRangePrefixTreeStrategy extends RecursivePrefixTreeStrategy {
/** Unsupported. */
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
index 0716e78..d65e9f5 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
@@ -25,7 +25,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
@@ -180,7 +180,7 @@ public abstract class PrefixTreeStrategy extends SpatialStrategy {
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
PointPrefixTreeFieldCacheProvider p = provider.get( getFieldName() );
if( p == null ) {
synchronized (this) {//double checked locking idiom is okay since provider is threadsafe
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
index 47ac90e..3a33380 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
@@ -22,29 +22,27 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
-import java.util.Map;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.util.DistanceToShapeValueSource;
-import org.apache.lucene.spatial.util.ShapePredicateValueSource;
+import org.apache.lucene.spatial.util.ShapeValuesPredicate;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.io.BinaryCodec;
import org.locationtech.spatial4j.shape.Point;
@@ -100,7 +98,7 @@ public class SerializedDVStrategy extends SpatialStrategy {
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
//TODO if makeShapeValueSource gets lifted to the top; this could become a generic impl.
return new DistanceToShapeValueSource(makeShapeValueSource(), queryPoint, multiplier, ctx);
}
@@ -111,18 +109,15 @@ public class SerializedDVStrategy extends SpatialStrategy {
*/
@Override
public Query makeQuery(SpatialArgs args) {
- ValueSource shapeValueSource = makeShapeValueSource();
- ShapePredicateValueSource predicateValueSource = new ShapePredicateValueSource(
- shapeValueSource, args.getOperation(), args.getShape());
+ ShapeValuesSource shapeValueSource = makeShapeValueSource();
+ ShapeValuesPredicate predicateValueSource = new ShapeValuesPredicate(shapeValueSource, args.getOperation(), args.getShape());
return new PredicateValueSourceQuery(predicateValueSource);
}
/**
- * Provides access to each shape per document as a ValueSource in which
- * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)} returns a {@link
- * Shape}.
+ * Provides access to each shape per document
*/ //TODO raise to SpatialStrategy
- public ValueSource makeShapeValueSource() {
+ public ShapeValuesSource makeShapeValueSource() {
return new ShapeDocValueSource(getFieldName(), ctx.getBinaryCodec());
}
@@ -130,9 +125,9 @@ public class SerializedDVStrategy extends SpatialStrategy {
* by {@link TwoPhaseIterator}.
*/
static class PredicateValueSourceQuery extends Query {
- private final ValueSource predicateValueSource;//we call boolVal(doc)
+ private final ShapeValuesPredicate predicateValueSource;
- public PredicateValueSourceQuery(ValueSource predicateValueSource) {
+ public PredicateValueSourceQuery(ShapeValuesPredicate predicateValueSource) {
this.predicateValueSource = predicateValueSource;
}
@@ -142,21 +137,8 @@ public class SerializedDVStrategy extends SpatialStrategy {
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
- final FunctionValues predFuncValues = predicateValueSource.getValues(null, context);
- return new ConstantScoreScorer(this, score(), new TwoPhaseIterator(approximation) {
-
- @Override
- public boolean matches() throws IOException {
- final int docID = approximation.docID();
- return predFuncValues.boolVal(docID);
- }
-
- @Override
- public float matchCost() {
- // TODO: what is the cost of the predicateValueSource
- return 100f;
- }
- });
+ TwoPhaseIterator it = predicateValueSource.iterator(context, approximation);
+ return new ConstantScoreScorer(this, score(), it);
}
};
}
@@ -184,7 +166,7 @@ public class SerializedDVStrategy extends SpatialStrategy {
* Implements a ValueSource by deserializing a Shape in from BinaryDocValues using BinaryCodec.
* @see #makeShapeValueSource()
*/
- static class ShapeDocValueSource extends ValueSource {
+ static class ShapeDocValueSource extends ShapeValuesSource {
private final String fieldName;
private final BinaryCodec binaryCodec;//spatial4j
@@ -195,65 +177,21 @@ public class SerializedDVStrategy extends SpatialStrategy {
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public ShapeValues getValues(LeafReaderContext readerContext) throws IOException {
final BinaryDocValues docValues = readerContext.reader().getBinaryDocValues(fieldName);
- return new FunctionValues() {
- int bytesRefDoc = -1;
- BytesRefBuilder bytesRef = new BytesRefBuilder();
-
- boolean fillBytes(int doc) throws IOException {
- if (bytesRefDoc != doc) {
- if (docValues.docID() < doc) {
- docValues.advance(doc);
- }
- if (docValues.docID() == doc) {
- bytesRef.copyBytes(docValues.binaryValue());
- } else {
- bytesRef.clear();
- }
- bytesRefDoc = doc;
- }
- return bytesRef.length() != 0;
- }
-
- @Override
- public boolean exists(int doc) throws IOException {
- return fillBytes(doc);
- }
-
- @Override
- public boolean bytesVal(int doc, BytesRefBuilder target) throws IOException {
- target.clear();
- if (fillBytes(doc)) {
- target.copyBytes(bytesRef);
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public Object objectVal(int docId) throws IOException {
- if (!fillBytes(docId))
- return null;
- DataInputStream dataInput = new DataInputStream(
- new ByteArrayInputStream(bytesRef.bytes(), 0, bytesRef.length()));
- try {
- return binaryCodec.readShape(dataInput);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
+ return new ShapeValues() {
@Override
- public Explanation explain(int doc) throws IOException {
- return Explanation.match(Float.NaN, toString(doc));
+ public boolean advanceExact(int doc) throws IOException {
+ return docValues.advanceExact(doc);
}
@Override
- public String toString(int doc) throws IOException {
- return description() + "=" + objectVal(doc);//TODO truncate?
+ public Shape value() throws IOException {
+ BytesRef bytesRef = docValues.binaryValue();
+ DataInputStream dataInput
+ = new DataInputStream(new ByteArrayInputStream(bytesRef.bytes, bytesRef.offset, bytesRef.length));
+ return binaryCodec.readShape(dataInput);
}
};
@@ -278,7 +216,7 @@ public class SerializedDVStrategy extends SpatialStrategy {
}
@Override
- public String description() {
+ public String toString() {
return "shapeDocVal(" + fieldName + ")";
}
}//ShapeDocValueSource
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
index d43d4e8..8d2f6f9 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
@@ -16,65 +16,74 @@
*/
package org.apache.lucene.spatial.util;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.Explanation;
+
/**
* Caches the doubleVal of another value source in a HashMap
* so that it is computed only once.
* @lucene.internal
*/
-public class CachingDoubleValueSource extends ValueSource {
+public class CachingDoubleValueSource extends DoubleValuesSource {
- final ValueSource source;
+ final DoubleValuesSource source;
final Map<Integer, Double> cache;
- public CachingDoubleValueSource( ValueSource source )
- {
+ public CachingDoubleValueSource(DoubleValuesSource source) {
this.source = source;
cache = new HashMap<>();
}
@Override
- public String description() {
- return "Cached["+source.description()+"]";
+ public String toString() {
+ return "Cached["+source.toString()+"]";
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
final int base = readerContext.docBase;
- final FunctionValues vals = source.getValues(context,readerContext);
- return new FunctionValues() {
+ final DoubleValues vals = source.getValues(readerContext, scores);
+ return new DoubleValues() {
@Override
- public double doubleVal(int doc) throws IOException {
- Integer key = Integer.valueOf( base+doc );
- Double v = cache.get( key );
- if( v == null ) {
- v = Double.valueOf( vals.doubleVal(doc) );
- cache.put( key, v );
+ public double doubleValue() throws IOException {
+ int key = base + doc;
+ Double v = cache.get(key);
+ if (v == null) {
+ v = vals.doubleValue();
+ cache.put(key, v);
}
- return v.doubleValue();
+ return v;
}
@Override
- public float floatVal(int doc) throws IOException {
- return (float)doubleVal(doc);
+ public boolean advanceExact(int doc) throws IOException {
+ this.doc = doc;
+ return vals.advanceExact(doc);
}
- @Override
- public String toString(int doc) throws IOException {
- return doubleVal(doc)+"";
- }
+ int doc = -1;
+
};
}
@Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
+ return source.explain(ctx, docId, scoreExplanation);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
index 6993adb..a926d7e 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/DistanceToShapeValueSource.java
@@ -17,81 +17,67 @@
package org.apache.lucene.spatial.util;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
-
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.distance.DistanceCalculator;
import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Shape;
/**
- * The distance from a provided Point to a Point retrieved from a ValueSource via
- * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)}. The distance
+ * The distance from a provided Point to a Point retrieved from an ShapeValuesSource. The distance
* is calculated via a {@link org.locationtech.spatial4j.distance.DistanceCalculator}.
*
* @lucene.experimental
*/
-public class DistanceToShapeValueSource extends ValueSource {
- private final ValueSource shapeValueSource;
+public class DistanceToShapeValueSource extends DoubleValuesSource {
+
+ private final ShapeValuesSource shapeValueSource;
private final Point queryPoint;
private final double multiplier;
private final DistanceCalculator distCalc;
- //TODO if FunctionValues returns NaN; will things be ok?
- private final double nullValue;//computed
+ //TODO if DoubleValues returns NaN; will things be ok?
+ private final double nullValue;
- public DistanceToShapeValueSource(ValueSource shapeValueSource, Point queryPoint,
+ public DistanceToShapeValueSource(ShapeValuesSource shapeValueSource, Point queryPoint,
double multiplier, SpatialContext ctx) {
this.shapeValueSource = shapeValueSource;
this.queryPoint = queryPoint;
this.multiplier = multiplier;
this.distCalc = ctx.getDistCalc();
- this.nullValue =
- (ctx.isGeo() ? 180 * multiplier : Double.MAX_VALUE);
+ this.nullValue = (ctx.isGeo() ? 180 * multiplier : Double.MAX_VALUE);
}
@Override
- public String description() {
- return "distance(" + queryPoint + " to " + shapeValueSource.description() + ")*" + multiplier + ")";
+ public String toString() {
+ return "distance(" + queryPoint + " to " + shapeValueSource.toString() + ")*" + multiplier + ")";
}
@Override
- public void createWeight(Map context, IndexSearcher searcher) throws IOException {
- shapeValueSource.createWeight(context, searcher);
- }
+ public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
- @Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final FunctionValues shapeValues = shapeValueSource.getValues(context, readerContext);
+ final ShapeValues shapeValues = shapeValueSource.getValues(readerContext);
- return new DoubleDocValues(this) {
+ return DoubleValues.withDefault(new DoubleValues() {
@Override
- public double doubleVal(int doc) throws IOException {
- Shape shape = (Shape) shapeValues.objectVal(doc);
- if (shape == null || shape.isEmpty())
- return nullValue;
- Point pt = shape.getCenter();
- return distCalc.distance(queryPoint, pt) * multiplier;
+ public double doubleValue() throws IOException {
+ return distCalc.distance(queryPoint, shapeValues.value().getCenter()) * multiplier;
}
@Override
- public Explanation explain(int doc) throws IOException {
- Explanation exp = super.explain(doc);
- List<Explanation> details = new ArrayList<>(Arrays.asList(exp.getDetails()));
- details.add(shapeValues.explain(doc));
- return Explanation.match(exp.getValue(), exp.getDescription(), details);
+ public boolean advanceExact(int doc) throws IOException {
+ return shapeValues.advanceExact(doc);
}
- };
+ }, nullValue);
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ReciprocalDoubleValuesSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ReciprocalDoubleValuesSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ReciprocalDoubleValuesSource.java
new file mode 100644
index 0000000..f0e2c45
--- /dev/null
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ReciprocalDoubleValuesSource.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.spatial.util;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Transforms a DoubleValuesSource using the formula v = k / (v + k)
+ */
+public class ReciprocalDoubleValuesSource extends DoubleValuesSource {
+
+ private final double distToEdge;
+ private final DoubleValuesSource input;
+
+ /**
+ * Creates a ReciprocalDoubleValuesSource
+ * @param distToEdge the value k in v = k / (v + k)
+ * @param input the input DoubleValuesSource to transform
+ */
+ public ReciprocalDoubleValuesSource(double distToEdge, DoubleValuesSource input) {
+ this.distToEdge = distToEdge;
+ this.input = input;
+ }
+
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ DoubleValues in = input.getValues(ctx, scores);
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return recip(in.doubleValue());
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return in.advanceExact(doc);
+ }
+ };
+ }
+
+ private double recip(double in) {
+ return distToEdge / (in + distToEdge);
+ }
+
+ @Override
+ public boolean needsScores() {
+ return input.needsScores();
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
+ Explanation expl = input.explain(ctx, docId, scoreExplanation);
+ return Explanation.match((float)recip(expl.getValue()),
+ distToEdge + " / (v + " + distToEdge + "), computed from:", expl);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ReciprocalDoubleValuesSource that = (ReciprocalDoubleValuesSource) o;
+ return Double.compare(that.distToEdge, distToEdge) == 0 &&
+ Objects.equals(input, that.input);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(distToEdge, input);
+ }
+
+ @Override
+ public String toString() {
+ return "recip(" + distToEdge + ", " + input.toString() + ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
index 9c2c0e3..6767991 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeAreaValueSource.java
@@ -17,36 +17,29 @@
package org.apache.lucene.spatial.util;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.shape.Shape;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
-
/**
- * The area of a Shape retrieved from a ValueSource via
- * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)}.
+ * The area of a Shape retrieved from an ShapeValuesSource
*
* @see Shape#getArea(org.locationtech.spatial4j.context.SpatialContext)
*
* @lucene.experimental
*/
-public class ShapeAreaValueSource extends ValueSource {
- private final ValueSource shapeValueSource;
+public class ShapeAreaValueSource extends DoubleValuesSource {
+ private final ShapeValuesSource shapeValueSource;
private final SpatialContext ctx;//not part of identity; should be associated with shapeValueSource indirectly
private final boolean geoArea;
private double multiplier;
- public ShapeAreaValueSource(ValueSource shapeValueSource, SpatialContext ctx, boolean geoArea, double multiplier) {
+ public ShapeAreaValueSource(ShapeValuesSource shapeValueSource, SpatialContext ctx, boolean geoArea, double multiplier) {
this.shapeValueSource = shapeValueSource;
this.ctx = ctx;
this.geoArea = geoArea;
@@ -54,43 +47,29 @@ public class ShapeAreaValueSource extends ValueSource {
}
@Override
- public String description() {
- return "area(" + shapeValueSource.description() + ",geo=" + geoArea + ")";
- }
-
- @Override
- public void createWeight(Map context, IndexSearcher searcher) throws IOException {
- shapeValueSource.createWeight(context, searcher);
+ public String toString() {
+ return "area(" + shapeValueSource.toString() + ",geo=" + geoArea + ")";
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final FunctionValues shapeValues = shapeValueSource.getValues(context, readerContext);
-
- return new DoubleDocValues(this) {
+ public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
+ final ShapeValues shapeValues = shapeValueSource.getValues(readerContext);
+ return DoubleValues.withDefault(new DoubleValues() {
@Override
- public double doubleVal(int doc) throws IOException {
- Shape shape = (Shape) shapeValues.objectVal(doc);
- if (shape == null || shape.isEmpty())
- return 0;//or NaN?
- //This part of Spatial4j API is kinda weird. Passing null means 2D area, otherwise geo
- // assuming ctx.isGeo()
- return shape.getArea( geoArea ? ctx : null ) * multiplier;
+ public double doubleValue() throws IOException {
+ return shapeValues.value().getArea(geoArea ? ctx : null) * multiplier;
}
@Override
- public boolean exists(int doc) throws IOException {
- return shapeValues.exists(doc);
+ public boolean advanceExact(int doc) throws IOException {
+ return shapeValues.advanceExact(doc);
}
+ }, 0);
+ }
- @Override
- public Explanation explain(int doc) throws IOException {
- Explanation exp = super.explain(doc);
- List<Explanation> details = new ArrayList<>(Arrays.asList(exp.getDetails()));
- details.add(shapeValues.explain(doc));
- return Explanation.match(exp.getValue(), exp.getDescription(), details);
- }
- };
+ @Override
+ public boolean needsScores() {
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java
index 1ac84e8..2d7ff4a 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeFieldCacheDistanceValueSource.java
@@ -16,26 +16,25 @@
*/
package org.apache.lucene.spatial.util;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.distance.DistanceCalculator;
import org.locationtech.spatial4j.shape.Point;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
/**
- * An implementation of the Lucene ValueSource that returns the spatial distance
+ * A DoubleValuesSource that returns the spatial distance
* between an input point and a document's points in
* {@link ShapeFieldCacheProvider}. The shortest distance is returned if a
* document has more than one point.
*
* @lucene.internal
*/
-public class ShapeFieldCacheDistanceValueSource extends ValueSource {
+public class ShapeFieldCacheDistanceValueSource extends DoubleValuesSource {
private final SpatialContext ctx;
private final Point from;
@@ -51,43 +50,43 @@ public class ShapeFieldCacheDistanceValueSource extends ValueSource {
}
@Override
- public String description() {
+ public String toString() {
return getClass().getSimpleName()+"("+provider+", "+from+")";
}
@Override
- public FunctionValues getValues(Map context, final LeafReaderContext readerContext) throws IOException {
- return new FunctionValues() {
+ public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
+
+ final double nullValue = (ctx.isGeo() ? 180 * multiplier : Double.MAX_VALUE);
+
+ return DoubleValues.withDefault(new DoubleValues() {
private final ShapeFieldCache<Point> cache =
provider.getCache(readerContext.reader());
private final Point from = ShapeFieldCacheDistanceValueSource.this.from;
private final DistanceCalculator calculator = ctx.getDistCalc();
- private final double nullValue = (ctx.isGeo() ? 180 * multiplier : Double.MAX_VALUE);
- @Override
- public float floatVal(int doc) {
- return (float) doubleVal(doc);
- }
+ private List<Point> currentVals;
@Override
- public double doubleVal(int doc) {
-
- List<Point> vals = cache.getShapes( doc );
- if( vals != null ) {
- double v = calculator.distance(from, vals.get(0));
- for( int i=1; i<vals.size(); i++ ) {
- v = Math.min(v, calculator.distance(from, vals.get(i)));
- }
- return v * multiplier;
+ public double doubleValue() throws IOException {
+ double v = calculator.distance(from, currentVals.get(0));
+ for (int i = 1; i < currentVals.size(); i++) {
+ v = Math.min(v, calculator.distance(from, currentVals.get(i)));
}
- return nullValue;
+ return v * multiplier;
}
@Override
- public String toString(int doc) {
- return description() + "=" + floatVal(doc);
+ public boolean advanceExact(int doc) throws IOException {
+ currentVals = cache.getShapes(doc);
+ return currentVals != null;
}
- };
+ }, nullValue);
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
deleted file mode 100644
index d0dc51f..0000000
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapePredicateValueSource.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.spatial.util;
-
-import org.locationtech.spatial4j.shape.Shape;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.BoolDocValues;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.spatial.query.SpatialOperation;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A boolean ValueSource that compares a shape from a provided ValueSource with a given Shape and sees
- * if it matches a given {@link SpatialOperation} (the predicate).
- *
- * @lucene.experimental
- */
-public class ShapePredicateValueSource extends ValueSource {
- private final ValueSource shapeValuesource;//the left hand side
- private final SpatialOperation op;
- private final Shape queryShape;//the right hand side (constant)
-
- /**
- *
- * @param shapeValuesource Must yield {@link Shape} instances from its objectVal(doc). If null
- * then the result is false. This is the left-hand (indexed) side.
- * @param op the predicate
- * @param queryShape The shape on the right-hand (query) side.
- */
- public ShapePredicateValueSource(ValueSource shapeValuesource, SpatialOperation op, Shape queryShape) {
- this.shapeValuesource = shapeValuesource;
- this.op = op;
- this.queryShape = queryShape;
- }
-
- @Override
- public String description() {
- return shapeValuesource + " " + op + " " + queryShape;
- }
-
- @Override
- public void createWeight(Map context, IndexSearcher searcher) throws IOException {
- shapeValuesource.createWeight(context, searcher);
- }
-
- @Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final FunctionValues shapeValues = shapeValuesource.getValues(context, readerContext);
-
- return new BoolDocValues(this) {
- @Override
- public boolean boolVal(int doc) throws IOException {
- Shape indexedShape = (Shape) shapeValues.objectVal(doc);
- if (indexedShape == null)
- return false;
- return op.evaluate(indexedShape, queryShape);
- }
-
- @Override
- public Explanation explain(int doc) throws IOException {
- Explanation exp = super.explain(doc);
- List<Explanation> details = new ArrayList<>(Arrays.asList(exp.getDetails()));
- details.add(shapeValues.explain(doc));
- return Explanation.match(exp.getValue(), exp.getDescription(), details);
- }
- };
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- ShapePredicateValueSource that = (ShapePredicateValueSource) o;
-
- if (!shapeValuesource.equals(that.shapeValuesource)) return false;
- if (!op.equals(that.op)) return false;
- if (!queryShape.equals(that.queryShape)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = shapeValuesource.hashCode();
- result = 31 * result + op.hashCode();
- result = 31 * result + queryShape.hashCode();
- return result;
- }
-}
[04/18] lucene-solr:feature/autoscaling: SOLR-10272: More detailed
error message, explaining what to do
Posted by sh...@apache.org.
SOLR-10272: More detailed error message, explaining what to do
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e6f48ca6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e6f48ca6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e6f48ca6
Branch: refs/heads/feature/autoscaling
Commit: e6f48ca6bcea8962282b82826709fbfd3c2fa6bb
Parents: 9c75c80
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Tue Jun 27 18:16:39 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Tue Jun 27 18:23:15 2017 +0530
----------------------------------------------------------------------
.../src/test-files/solr/configsets/_default/conf/solrconfig.xml | 2 +-
.../src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e6f48ca6/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
index acf95ba..09d8e5a 100644
--- a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
@@ -24,7 +24,7 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<requestHandler name="my_error_handler" class="solr.ThrowErrorOnInitRequestHandler">
- <str name="error">This is the _default configset, which is designed to throw error upon collection creation.</str>
+ <str name="error">This is the _default configset, which is designed to throw error upon collection creation from unit tests. If you see this error while running a unit test, you might want to explicitly specify the configset (when creating a collection) you intend to use (usually named "conf1" or "conf" etc.).</str>
</requestHandler>
<schemaFactory class="ClassicIndexSchemaFactory"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e6f48ca6/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 9c414cc..a4ed93f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -71,7 +71,7 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
System.out.println("Errors are: "+response.getErrorMessages());
assertTrue(response.getErrorMessages() != null && response.getErrorMessages().size() > 0);
assertTrue(response.getErrorMessages().getVal(0).contains("This is the _default configset, which is designed"
- + " to throw error upon collection creation."));
+ + " to throw error upon collection creation"));
}
@Test
[10/18] lucene-solr:feature/autoscaling: SOLR-10272: Fix precommit
(the correct way)
Posted by sh...@apache.org.
SOLR-10272: Fix precommit (the correct way)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab319eab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab319eab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab319eab
Branch: refs/heads/feature/autoscaling
Commit: ab319eab2427e0da6979c78cd391de2c49da1259
Parents: 2f2e00f
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Tue Jun 27 21:59:32 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Tue Jun 27 21:59:32 2017 +0530
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/cloud/ZkController.java | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab319eab/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index f8543db..01e1693 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -50,7 +50,6 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
-import org.apache.lucene.util.SuppressForbidden;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
@@ -716,10 +715,8 @@ public class ZkController {
return configDirPath;
}
- @SuppressForbidden(reason = "This will be loaded for unit tests, and hence an exception can be made. "
- + "In regular server operation, there shouldn't be a situation where we reach this point.")
private static String getDefaultConfigDirFromClasspath(String serverSubPath) {
- URL classpathUrl = Thread.currentThread().getContextClassLoader().getResource(serverSubPath);
+ URL classpathUrl = ZkController.class.getClassLoader().getResource(serverSubPath);
try {
if (classpathUrl != null && new File(classpathUrl.toURI()).exists()) {
return new File(classpathUrl.toURI()).getAbsolutePath();
[09/18] lucene-solr:feature/autoscaling: LUCENE-7723: Add hashCode
and equals to DoubleValuesSource
Posted by sh...@apache.org.
LUCENE-7723: Add hashCode and equals to DoubleValuesSource
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1a278ae8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1a278ae8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1a278ae8
Branch: refs/heads/feature/autoscaling
Commit: 1a278ae89bee3072803abe2387091df9942e96af
Parents: 808171a
Author: Alan Woodward <ro...@apache.org>
Authored: Mon May 8 11:24:17 2017 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Tue Jun 27 15:53:39 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../lucene/search/DoubleValuesSource.java | 209 ++++++++++---------
.../apache/lucene/search/LongValuesSource.java | 82 ++++++--
.../lucene/search/TestDoubleValuesSource.java | 16 +-
.../facet/range/TestRangeFacetCounts.java | 75 ++++---
.../queries/function/FunctionScoreQuery.java | 9 -
.../lucene/queries/function/ValueSource.java | 197 ++++++++++-------
.../function/TestFunctionScoreExplanations.java | 8 +-
.../function/TestFunctionScoreQuery.java | 91 +++++++-
.../DocumentValueSourceDictionaryTest.java | 15 ++
10 files changed, 459 insertions(+), 246 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index eede65b..517d937 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -89,6 +89,9 @@ API Changes
* LUCENE-7867: The deprecated Token class is now only available in the test
framework (Alan Woodward, Adrien Grand)
+* LUCENE-7723: DoubleValuesSource enforces implementation of equals() and
+ hashCode() (Alan Woodward)
+
Bug Fixes
* LUCENE-7626: IndexWriter will no longer accept broken token offsets
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
index 929441d..342c08b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
@@ -20,9 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Objects;
import java.util.function.DoubleToLongFunction;
-import java.util.function.DoubleUnaryOperator;
import java.util.function.LongToDoubleFunction;
-import java.util.function.ToDoubleBiFunction;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
@@ -76,14 +74,33 @@ public abstract class DoubleValuesSource {
return new DoubleValuesSortField(this, reverse);
}
+ @Override
+ public abstract int hashCode();
+
+ @Override
+ public abstract boolean equals(Object obj);
+
+ @Override
+ public abstract String toString();
+
/**
* Convert to a LongValuesSource by casting the double values to longs
*/
public final LongValuesSource toLongValuesSource() {
- return new LongValuesSource() {
- @Override
+ return new LongDoubleValuesSource(this);
+ }
+
+ private static class LongDoubleValuesSource extends LongValuesSource {
+
+ private final DoubleValuesSource inner;
+
+ private LongDoubleValuesSource(DoubleValuesSource inner) {
+ this.inner = inner;
+ }
+
+ @Override
public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- DoubleValues in = DoubleValuesSource.this.getValues(ctx, scores);
+ DoubleValues in = inner.getValues(ctx, scores);
return new LongValues() {
@Override
public long longValue() throws IOException {
@@ -99,9 +116,27 @@ public abstract class DoubleValuesSource {
@Override
public boolean needsScores() {
- return DoubleValuesSource.this.needsScores();
+ return inner.needsScores();
}
- };
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ LongDoubleValuesSource that = (LongDoubleValuesSource) o;
+ return Objects.equals(inner, that.inner);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(inner);
+ }
+
+ @Override
+ public String toString() {
+ return "long(" + inner.toString() + ")";
+ }
+
}
/**
@@ -164,115 +199,80 @@ public abstract class DoubleValuesSource {
public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) {
return scoreExplanation;
}
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj == this;
+ }
+
+ @Override
+ public String toString() {
+ return "scores";
+ }
};
/**
* Creates a DoubleValuesSource that always returns a constant value
*/
public static DoubleValuesSource constant(double value) {
- return new DoubleValuesSource() {
- @Override
- public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- return new DoubleValues() {
- @Override
- public double doubleValue() throws IOException {
- return value;
- }
-
- @Override
- public boolean advanceExact(int doc) throws IOException {
- return true;
- }
- };
- }
-
- @Override
- public boolean needsScores() {
- return false;
- }
+ return new ConstantValuesSource(value);
+ }
- @Override
- public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) {
- return Explanation.match((float) value, "constant(" + value + ")");
- }
+ private static class ConstantValuesSource extends DoubleValuesSource {
- @Override
- public String toString() {
- return "constant(" + value + ")";
- }
- };
- }
+ private final double value;
- /**
- * Creates a DoubleValuesSource that is a function of another DoubleValuesSource
- */
- public static DoubleValuesSource function(DoubleValuesSource in, String description, DoubleUnaryOperator function) {
- return new DoubleValuesSource() {
- @Override
- public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- DoubleValues inputs = in.getValues(ctx, scores);
- return new DoubleValues() {
- @Override
- public double doubleValue() throws IOException {
- return function.applyAsDouble(inputs.doubleValue());
- }
+ private ConstantValuesSource(double value) {
+ this.value = value;
+ }
- @Override
- public boolean advanceExact(int doc) throws IOException {
- return inputs.advanceExact(doc);
- }
- };
- }
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return value;
+ }
- @Override
- public boolean needsScores() {
- return in.needsScores();
- }
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return true;
+ }
+ };
+ }
- @Override
- public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
- Explanation inner = in.explain(ctx, docId, scoreExplanation);
- return Explanation.match((float) function.applyAsDouble(inner.getValue()), description + ", computed from:", inner, scoreExplanation);
- }
- };
- }
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
- /**
- * Creates a DoubleValuesSource that is a function of another DoubleValuesSource and a score
- * @param in the DoubleValuesSource to use as an input
- * @param description a description of the function
- * @param function a function of the form (source, score) == result
- */
- public static DoubleValuesSource scoringFunction(DoubleValuesSource in, String description, ToDoubleBiFunction<Double, Double> function) {
- return new DoubleValuesSource() {
- @Override
- public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- DoubleValues inputs = in.getValues(ctx, scores);
- return new DoubleValues() {
- @Override
- public double doubleValue() throws IOException {
- return function.applyAsDouble(inputs.doubleValue(), scores.doubleValue());
- }
+ @Override
+ public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) {
+ return Explanation.match((float) value, "constant(" + value + ")");
+ }
- @Override
- public boolean advanceExact(int doc) throws IOException {
- return inputs.advanceExact(doc);
- }
- };
- }
+ @Override
+ public int hashCode() {
+ return Objects.hash(value);
+ }
- @Override
- public boolean needsScores() {
- return true;
- }
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ConstantValuesSource that = (ConstantValuesSource) o;
+ return Double.compare(that.value, value) == 0;
+ }
- @Override
- public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
- Explanation inner = in.explain(ctx, docId, scoreExplanation);
- return Explanation.match((float) function.applyAsDouble((double)inner.getValue(), (double)scoreExplanation.getValue()),
- description + ", computed from:", inner, scoreExplanation);
- }
- };
+ @Override
+ public String toString() {
+ return "constant(" + value + ")";
+ }
}
/**
@@ -313,6 +313,11 @@ public abstract class DoubleValuesSource {
}
@Override
+ public String toString() {
+ return "double(" + field + ")";
+ }
+
+ @Override
public int hashCode() {
return Objects.hash(field, decoder);
}
@@ -342,9 +347,9 @@ public abstract class DoubleValuesSource {
public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
DoubleValues values = getValues(ctx, null);
if (values.advanceExact(docId))
- return Explanation.match((float)values.doubleValue(), "double(" + field + ")");
+ return Explanation.match((float)values.doubleValue(), this.toString());
else
- return Explanation.noMatch("double(" + field + ")");
+ return Explanation.noMatch(this.toString());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/core/src/java/org/apache/lucene/search/LongValuesSource.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/LongValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/LongValuesSource.java
index 9a23cab..975a905 100644
--- a/lucene/core/src/java/org/apache/lucene/search/LongValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/LongValuesSource.java
@@ -52,6 +52,15 @@ public abstract class LongValuesSource {
*/
public abstract boolean needsScores();
+ @Override
+ public abstract int hashCode();
+
+ @Override
+ public abstract boolean equals(Object obj);
+
+ @Override
+ public abstract String toString();
+
/**
* Create a sort field based on the value of this producer
* @param reverse true if the sort should be decreasing
@@ -78,27 +87,55 @@ public abstract class LongValuesSource {
* Creates a LongValuesSource that always returns a constant value
*/
public static LongValuesSource constant(long value) {
- return new LongValuesSource() {
- @Override
- public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- return new LongValues() {
- @Override
- public long longValue() throws IOException {
- return value;
- }
-
- @Override
- public boolean advanceExact(int doc) throws IOException {
- return true;
- }
- };
- }
+ return new ConstantLongValuesSource(value);
+ }
+
+ private static class ConstantLongValuesSource extends LongValuesSource {
+
+ private final long value;
+
+ private ConstantLongValuesSource(long value) {
+ this.value = value;
+ }
+
+ @Override
+ public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ return new LongValues() {
+ @Override
+ public long longValue() throws IOException {
+ return value;
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return true;
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(value);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ConstantLongValuesSource that = (ConstantLongValuesSource) o;
+ return value == that.value;
+ }
+
+ @Override
+ public String toString() {
+ return "constant(" + value + ")";
+ }
- @Override
- public boolean needsScores() {
- return false;
- }
- };
}
private static class FieldValuesSource extends LongValuesSource {
@@ -118,6 +155,11 @@ public abstract class LongValuesSource {
}
@Override
+ public String toString() {
+ return "long(" + field + ")";
+ }
+
+ @Override
public int hashCode() {
return Objects.hash(field);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
index 54501e6..de2479c 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
@@ -92,6 +92,14 @@ public class TestDoubleValuesSource extends LuceneTestCase {
assertEquals(vs1.hashCode(), vs2.hashCode());
DoubleValuesSource v3 = DoubleValuesSource.fromLongField("long");
assertFalse(vs1.equals(v3));
+
+ assertEquals(DoubleValuesSource.constant(5), DoubleValuesSource.constant(5));
+ assertEquals(DoubleValuesSource.constant(5).hashCode(), DoubleValuesSource.constant(5).hashCode());
+ assertFalse((DoubleValuesSource.constant(5).equals(DoubleValuesSource.constant(6))));
+
+ assertEquals(DoubleValuesSource.SCORES, DoubleValuesSource.SCORES);
+ assertFalse(DoubleValuesSource.constant(5).equals(DoubleValuesSource.SCORES));
+
}
public void testSimpleFieldSortables() throws Exception {
@@ -184,13 +192,6 @@ public class TestDoubleValuesSource extends LuceneTestCase {
testExplanations(q, DoubleValuesSource.fromDoubleField("double"));
testExplanations(q, DoubleValuesSource.fromDoubleField("onefield"));
testExplanations(q, DoubleValuesSource.constant(5.45));
- testExplanations(q, DoubleValuesSource.function(
- DoubleValuesSource.fromDoubleField("double"), "v * 4 + 73",
- v -> v * 4 + 73
- ));
- testExplanations(q, DoubleValuesSource.scoringFunction(
- DoubleValuesSource.fromDoubleField("double"), "v * score", (v, s) -> v * s
- ));
}
}
@@ -227,4 +228,5 @@ public class TestDoubleValuesSource extends LuceneTestCase {
}
});
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
index 06bd2e5..5a80c3b 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
@@ -706,6 +706,51 @@ public class TestRangeFacetCounts extends FacetTestCase {
}
+ private static class PlusOneValuesSource extends DoubleValuesSource {
+
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ return new DoubleValues() {
+ int doc = -1;
+ @Override
+ public double doubleValue() throws IOException {
+ return doc + 1;
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ this.doc = doc;
+ return true;
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
+ return Explanation.match(docId + 1, "");
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj.getClass() == PlusOneValuesSource.class;
+ }
+
+ @Override
+ public String toString() {
+ return null;
+ }
+ }
+
public void testCustomDoubleValuesSource() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
@@ -718,35 +763,7 @@ public class TestRangeFacetCounts extends FacetTestCase {
// Test wants 3 docs in one segment:
writer.forceMerge(1);
- final DoubleValuesSource vs = new DoubleValuesSource() {
-
- @Override
- public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- return new DoubleValues() {
- int doc = -1;
- @Override
- public double doubleValue() throws IOException {
- return doc + 1;
- }
-
- @Override
- public boolean advanceExact(int doc) throws IOException {
- this.doc = doc;
- return true;
- }
- };
- }
-
- @Override
- public boolean needsScores() {
- return false;
- }
-
- @Override
- public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
- return Explanation.match(docId + 1, "");
- }
- };
+ final DoubleValuesSource vs = new PlusOneValuesSource();
FacetsConfig config = new FacetsConfig();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
index 63a98de..aa02c19 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionScoreQuery.java
@@ -121,15 +121,6 @@ public final class FunctionScoreQuery extends Query {
Explanation.match(boost, "boost"), expl);
}
- private Explanation scoreExplanation(LeafReaderContext context, int doc, DoubleValues scores) throws IOException {
- if (valueSource.needsScores() == false)
- return Explanation.match((float) scores.doubleValue(), valueSource.toString());
- float score = (float) scores.doubleValue();
- return Explanation.match(score, "computed from:",
- Explanation.match(score, valueSource.toString()),
- inner.explain(context, doc));
- }
-
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
Scorer in = inner.scorer(context);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
index e49c823..cb3faa3 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Map;
+import java.util.Objects;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSetIterator;
@@ -120,85 +121,141 @@ public abstract class ValueSource {
* Expose this ValueSource as a LongValuesSource
*/
public LongValuesSource asLongValuesSource() {
- return new LongValuesSource() {
- @Override
- public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- Map context = new IdentityHashMap<>();
- FakeScorer scorer = new FakeScorer();
- context.put("scorer", scorer);
- final FunctionValues fv = ValueSource.this.getValues(context, ctx);
- return new LongValues() {
-
- @Override
- public long longValue() throws IOException {
- return fv.longVal(scorer.current);
- }
+ return new WrappedLongValuesSource(this);
+ }
+
+ private static class WrappedLongValuesSource extends LongValuesSource {
+
+ private final ValueSource in;
+
+ private WrappedLongValuesSource(ValueSource in) {
+ this.in = in;
+ }
+
+ @Override
+ public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ Map context = new IdentityHashMap<>();
+ FakeScorer scorer = new FakeScorer();
+ context.put("scorer", scorer);
+ final FunctionValues fv = in.getValues(context, ctx);
+ return new LongValues() {
+
+ @Override
+ public long longValue() throws IOException {
+ return fv.longVal(scorer.current);
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ scorer.current = doc;
+ if (scores != null && scores.advanceExact(doc))
+ scorer.score = (float) scores.doubleValue();
+ else
+ scorer.score = 0;
+ return fv.exists(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ WrappedLongValuesSource that = (WrappedLongValuesSource) o;
+ return Objects.equals(in, that.in);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(in);
+ }
+
+ @Override
+ public String toString() {
+ return in.toString();
+ }
- @Override
- public boolean advanceExact(int doc) throws IOException {
- scorer.current = doc;
- if (scores != null && scores.advanceExact(doc))
- scorer.score = (float) scores.doubleValue();
- else
- scorer.score = 0;
- return fv.exists(doc);
- }
- };
- }
-
- @Override
- public boolean needsScores() {
- return false;
- }
- };
}
/**
* Expose this ValueSource as a DoubleValuesSource
*/
public DoubleValuesSource asDoubleValuesSource() {
- return new DoubleValuesSource() {
- @Override
- public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- Map context = new HashMap<>();
- FakeScorer scorer = new FakeScorer();
- context.put("scorer", scorer);
- FunctionValues fv = ValueSource.this.getValues(context, ctx);
- return new DoubleValues() {
-
- @Override
- public double doubleValue() throws IOException {
- return fv.doubleVal(scorer.current);
- }
+ return new WrappedDoubleValuesSource(this);
+ }
+
+ private static class WrappedDoubleValuesSource extends DoubleValuesSource {
+
+ private final ValueSource in;
- @Override
- public boolean advanceExact(int doc) throws IOException {
- scorer.current = doc;
- if (scores != null && scores.advanceExact(doc)) {
- scorer.score = (float) scores.doubleValue();
- }
- else
- scorer.score = 0;
- return fv.exists(doc);
+ private WrappedDoubleValuesSource(ValueSource in) {
+ this.in = in;
+ }
+
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ Map context = new HashMap<>();
+ FakeScorer scorer = new FakeScorer();
+ context.put("scorer", scorer);
+ FunctionValues fv = in.getValues(context, ctx);
+ return new DoubleValues() {
+
+ @Override
+ public double doubleValue() throws IOException {
+ return fv.doubleVal(scorer.current);
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ scorer.current = doc;
+ if (scores != null && scores.advanceExact(doc)) {
+ scorer.score = (float) scores.doubleValue();
}
- };
- }
-
- @Override
- public boolean needsScores() {
- return true; // be on the safe side
- }
-
- @Override
- public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
- Map context = new HashMap<>();
- FakeScorer scorer = new FakeScorer();
- scorer.score = scoreExplanation.getValue();
- context.put("scorer", scorer);
- FunctionValues fv = ValueSource.this.getValues(context, ctx);
- return fv.explain(docId);
- }
- };
+ else
+ scorer.score = 0;
+ return fv.exists(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return true; // be on the safe side
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
+ Map context = new HashMap<>();
+ FakeScorer scorer = new FakeScorer();
+ scorer.score = scoreExplanation.getValue();
+ context.put("scorer", scorer);
+ FunctionValues fv = in.getValues(context, ctx);
+ return fv.explain(docId);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ WrappedDoubleValuesSource that = (WrappedDoubleValuesSource) o;
+ return Objects.equals(in, that.in);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(in);
+ }
+
+ @Override
+ public String toString() {
+ return in.toString();
+ }
+
}
//
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java
index bc8dbc1..2ed9d72 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreExplanations.java
@@ -59,19 +59,15 @@ public class TestFunctionScoreExplanations extends BaseExplanationTestCase {
public void testExplanationsIncludingScore() throws Exception {
- DoubleValuesSource scores = DoubleValuesSource.function(DoubleValuesSource.SCORES, "v * 2", v -> v * 2);
-
Query q = new TermQuery(new Term(FIELD, "w1"));
- FunctionScoreQuery csq = new FunctionScoreQuery(q, scores);
+ FunctionScoreQuery csq = new FunctionScoreQuery(q, DoubleValuesSource.SCORES);
qtest(csq, new int[] { 0, 1, 2, 3 });
Explanation e1 = searcher.explain(q, 0);
Explanation e = searcher.explain(csq, 0);
- assertEquals(e.getDetails().length, 2);
-
- assertEquals(e1.getValue() * 2, e.getValue(), 0.00001);
+ assertEquals(e, e1);
}
public void testSubExplanations() throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
index de5ad88..431d5f8 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionScoreQuery.java
@@ -17,12 +17,18 @@
package org.apache.lucene.queries.function;
+import java.io.IOException;
+import java.util.function.DoubleUnaryOperator;
+import java.util.function.ToDoubleBiFunction;
+
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -70,7 +76,7 @@ public class TestFunctionScoreQuery extends FunctionTestSetup {
public void testScoreModifyingSource() throws Exception {
DoubleValuesSource iii = DoubleValuesSource.fromIntField("iii");
- DoubleValuesSource score = DoubleValuesSource.scoringFunction(iii, "v * s", (v, s) -> v * s);
+ DoubleValuesSource score = scoringFunction(iii, (v, s) -> v * s);
BooleanQuery bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(TEXT_FIELD, "first")), BooleanClause.Occur.SHOULD)
@@ -95,8 +101,7 @@ public class TestFunctionScoreQuery extends FunctionTestSetup {
// check boosts with non-distributive score source
public void testBoostsAreAppliedLast() throws Exception {
- DoubleValuesSource scores
- = DoubleValuesSource.function(DoubleValuesSource.SCORES, "ln(v + 4)", v -> Math.log(v + 4));
+ DoubleValuesSource scores = function(DoubleValuesSource.SCORES, v -> Math.log(v + 4));
Query q1 = new FunctionScoreQuery(new TermQuery(new Term(TEXT_FIELD, "text")), scores);
TopDocs plain = searcher.search(q1, 5);
@@ -111,4 +116,84 @@ public class TestFunctionScoreQuery extends FunctionTestSetup {
}
+ public static DoubleValuesSource function(DoubleValuesSource in, DoubleUnaryOperator function) {
+ return new DoubleValuesSource() {
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ DoubleValues v = in.getValues(ctx, scores);
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return function.applyAsDouble(v.doubleValue());
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return v.advanceExact(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return in.needsScores();
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "fn";
+ }
+ };
+ }
+
+ private static DoubleValuesSource scoringFunction(DoubleValuesSource in, ToDoubleBiFunction<Double, Double> function) {
+ return new DoubleValuesSource() {
+ @Override
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ DoubleValues v = in.getValues(ctx, scores);
+ return new DoubleValues() {
+ @Override
+ public double doubleValue() throws IOException {
+ return function.applyAsDouble(v.doubleValue(), scores.doubleValue());
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ return v.advanceExact(doc);
+ }
+ };
+ }
+
+ @Override
+ public boolean needsScores() {
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "fn";
+ }
+ };
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a278ae8/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
index 55970e4..fb91e20 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
@@ -171,6 +171,21 @@ public class DocumentValueSourceDictionaryTest extends LuceneTestCase {
public boolean needsScores() {
return false;
}
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return null;
+ }
};
}
[07/18] lucene-solr:feature/autoscaling: LUCENE-7737: Remove
spatial-extras dependency on queries module
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeValuesPredicate.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeValuesPredicate.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeValuesPredicate.java
new file mode 100644
index 0000000..e859d62
--- /dev/null
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/ShapeValuesPredicate.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.spatial.util;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
+import org.apache.lucene.spatial.query.SpatialOperation;
+import org.locationtech.spatial4j.shape.Shape;
+
+/**
+ * Compares a shape from a provided {@link ShapeValuesSource} with a given Shape and sees
+ * if it matches a given {@link SpatialOperation} (the predicate).
+ *
+ * Consumers should call {@link #iterator(LeafReaderContext, DocIdSetIterator)} to obtain a
+ * {@link TwoPhaseIterator} over a particular {@link DocIdSetIterator}. The initial DocIdSetIterator
+ * will be used as the approximation, and the {@link SpatialOperation} comparison will only be
+ * performed in {@link TwoPhaseIterator#matches()}
+ *
+ * @lucene.experimental
+ */
+public class ShapeValuesPredicate {
+ private final ShapeValuesSource shapeValuesource;//the left hand side
+ private final SpatialOperation op;
+ private final Shape queryShape;//the right hand side (constant)
+
+ /**
+ *
+ * @param shapeValuesource Must yield {@link Shape} instances from its objectVal(doc). If null
+ * then the result is false. This is the left-hand (indexed) side.
+ * @param op the predicate
+ * @param queryShape The shape on the right-hand (query) side.
+ */
+ public ShapeValuesPredicate(ShapeValuesSource shapeValuesource, SpatialOperation op, Shape queryShape) {
+ this.shapeValuesource = shapeValuesource;
+ this.op = op;
+ this.queryShape = queryShape;
+ }
+
+ @Override
+ public String toString() {
+ return shapeValuesource + " " + op + " " + queryShape;
+ }
+
+ public TwoPhaseIterator iterator(LeafReaderContext ctx, DocIdSetIterator approximation) throws IOException {
+ final ShapeValues shapeValues = shapeValuesource.getValues(ctx);
+ return new TwoPhaseIterator(approximation) {
+ @Override
+ public boolean matches() throws IOException {
+ return shapeValues.advanceExact(approximation.docID()) && op.evaluate(shapeValues.value(), queryShape);
+ }
+
+ @Override
+ public float matchCost() {
+ return 100; // is this necessary?
+ }
+ };
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ShapeValuesPredicate that = (ShapeValuesPredicate) o;
+
+ if (!shapeValuesource.equals(that.shapeValuesource)) return false;
+ if (!op.equals(that.op)) return false;
+ if (!queryShape.equals(that.queryShape)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = shapeValuesource.hashCode();
+ result = 31 * result + op.hashCode();
+ result = 31 * result + queryShape.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
index e1f8f7c..fc34000 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
@@ -16,29 +16,28 @@
*/
package org.apache.lucene.spatial.vector;
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
import org.locationtech.spatial4j.distance.DistanceCalculator;
import org.locationtech.spatial4j.shape.Point;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-
-import java.io.IOException;
-import java.util.Map;
/**
- * An implementation of the Lucene ValueSource model that returns the distance
- * for a {@link PointVectorStrategy}.
+ * A DoubleValuesSource that returns the distance for a {@link PointVectorStrategy}.
*
* @lucene.internal
*/
-public class DistanceValueSource extends ValueSource {
+public class DistanceValueSource extends DoubleValuesSource {
private PointVectorStrategy strategy;
private final Point from;
private final double multiplier;
+ private final double nullValue;
/**
* Constructor.
@@ -47,13 +46,14 @@ public class DistanceValueSource extends ValueSource {
this.strategy = strategy;
this.from = from;
this.multiplier = multiplier;
+ this.nullValue = 180 * multiplier;
}
/**
* Returns the ValueSource description.
*/
@Override
- public String description() {
+ public String toString() {
return "DistanceValueSource("+strategy+", "+from+")";
}
@@ -61,55 +61,35 @@ public class DistanceValueSource extends ValueSource {
* Returns the FunctionValues used by the function query.
*/
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
+ public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
LeafReader reader = readerContext.reader();
final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());
- return new FunctionValues() {
-
- private int lastDocID = -1;
+ return DoubleValues.withDefault(new DoubleValues() {
private final Point from = DistanceValueSource.this.from;
private final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();
- private final double nullValue =
- (strategy.getSpatialContext().isGeo() ? 180 * multiplier : Double.MAX_VALUE);
-
- private double getDocValue(NumericDocValues values, int doc) throws IOException {
- int curDocID = values.docID();
- if (doc > curDocID) {
- curDocID = values.advance(doc);
- }
- if (doc == curDocID) {
- return Double.longBitsToDouble(values.longValue());
- } else {
- return 0.0;
- }
- }
@Override
- public float floatVal(int doc) throws IOException {
- return (float) doubleVal(doc);
+ public double doubleValue() throws IOException {
+ double x = Double.longBitsToDouble(ptX.longValue());
+ double y = Double.longBitsToDouble(ptY.longValue());
+ return calculator.distance(from, x, y) * multiplier;
}
@Override
- public double doubleVal(int doc) throws IOException {
- // make sure it has minX and area
- double x = getDocValue(ptX, doc);
- if (ptX.docID() == doc) {
- double y = getDocValue(ptY, doc);
- assert ptY.docID() == doc;
- return calculator.distance(from, x, y) * multiplier;
- }
- return nullValue;
+ public boolean advanceExact(int doc) throws IOException {
+ return ptX.advanceExact(doc) && ptY.advanceExact(doc);
}
- @Override
- public String toString(int doc) throws IOException {
- return description() + "=" + floatVal(doc);
- }
- };
+ }, nullValue);
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
index ef3eaa4..868897d 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/vector/PointVectorStrategy.java
@@ -16,18 +16,30 @@
*/
package org.apache.lucene.spatial.vector;
+import java.io.IOException;
+import java.util.Objects;
+
import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.queries.function.FunctionRangeQuery;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
@@ -169,12 +181,12 @@ public class PointVectorStrategy extends SpatialStrategy {
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
return new DistanceValueSource(this, queryPoint, multiplier);
}
@Override
- public ConstantScoreQuery makeQuery(SpatialArgs args) {
+ public Query makeQuery(SpatialArgs args) {
if(! SpatialOperation.is( args.getOperation(),
SpatialOperation.Intersects,
SpatialOperation.IsWithin ))
@@ -186,13 +198,7 @@ public class PointVectorStrategy extends SpatialStrategy {
} else if (shape instanceof Circle) {
Circle circle = (Circle)shape;
Rectangle bbox = circle.getBoundingBox();
- Query approxQuery = makeWithin(bbox);
- BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();
- FunctionRangeQuery vsRangeQuery =
- new FunctionRangeQuery(makeDistanceValueSource(circle.getCenter()), 0.0, circle.getRadius(), true, true);
- bqBuilder.add(approxQuery, BooleanClause.Occur.FILTER);//should have lowest "cost" value; will drive iteration
- bqBuilder.add(vsRangeQuery, BooleanClause.Occur.FILTER);
- return new ConstantScoreQuery(bqBuilder.build());
+ return new DistanceRangeQuery(makeWithin(bbox), makeDistanceValueSource(circle.getCenter()), circle.getRadius());
} else {
throw new UnsupportedOperationException("Only Rectangles and Circles are currently supported, " +
"found [" + shape.getClass() + "]");//TODO
@@ -237,4 +243,71 @@ public class PointVectorStrategy extends SpatialStrategy {
//TODO try doc-value range query?
throw new UnsupportedOperationException("An index is required for this operation.");
}
+
+ private static class DistanceRangeQuery extends Query {
+
+ final Query inner;
+ final DoubleValuesSource distanceSource;
+ final double limit;
+
+ private DistanceRangeQuery(Query inner, DoubleValuesSource distanceSource, double limit) {
+ this.inner = inner;
+ this.distanceSource = distanceSource;
+ this.limit = limit;
+ }
+
+ @Override
+ public Query rewrite(IndexReader reader) throws IOException {
+ Query rewritten = inner.rewrite(reader);
+ if (rewritten == inner)
+ return this;
+ return new DistanceRangeQuery(rewritten, distanceSource, limit);
+ }
+
+ @Override
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
+ Weight w = inner.createWeight(searcher, needsScores, 1f);
+ return new ConstantScoreWeight(this, boost) {
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ Scorer in = w.scorer(context);
+ if (in == null)
+ return null;
+ DoubleValues v = distanceSource.getValues(context, DoubleValuesSource.fromScorer(in));
+ DocIdSetIterator approximation = in.iterator();
+ TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
+ @Override
+ public boolean matches() throws IOException {
+ return v.advanceExact(approximation.docID()) && v.doubleValue() <= limit;
+ }
+
+ @Override
+ public float matchCost() {
+ return 100; // distance calculation can be heavy!
+ }
+ };
+ return new ConstantScoreScorer(this, score(), twoPhase);
+ }
+ };
+ }
+
+ @Override
+ public String toString(String field) {
+ return "DistanceRangeQuery(" + inner.toString(field) + "; " + distanceSource.toString() + " < " + limit + ")";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ DistanceRangeQuery that = (DistanceRangeQuery) o;
+ return Objects.equals(inner, that.inner) &&
+ Objects.equals(distanceSource, that.distanceSource) && limit == that.limit;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(inner, distanceSource, limit);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
index 536436b..3e3b2e2 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
@@ -84,9 +84,6 @@ public class DistanceStrategyTest extends StrategyTestCase {
adoc("100", ctx.makePoint(2, 1));
adoc("101", ctx.makePoint(-1, 4));
adoc("103", (Shape)null);//test score for nothing
- adoc("999", ctx.makePoint(2, 1));//test deleted
- commit();
- deleteDoc("999");
commit();
//FYI distances are in docid order
checkDistValueSource(ctx.makePoint(4, 3), 2.8274937f, 5.0898066f, 180f);
@@ -100,9 +97,6 @@ public class DistanceStrategyTest extends StrategyTestCase {
Point p101 = ctx.makePoint(-1.001, 4.001);
adoc("101", p101);
adoc("103", (Shape)null);//test score for nothing
- adoc("999", ctx.makePoint(2, 1));//test deleted
- commit();
- deleteDoc("999");
commit();
double dist = ctx.getDistCalc().distance(p100, p101);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java
index 1faae21..ed4230f 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/SpatialExample.java
@@ -18,10 +18,6 @@ package org.apache.lucene.spatial;
import java.io.IOException;
-import org.locationtech.spatial4j.context.SpatialContext;
-import org.locationtech.spatial4j.distance.DistanceUtils;
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Shape;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
@@ -30,7 +26,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
@@ -46,6 +42,10 @@ import org.apache.lucene.spatial.query.SpatialOperation;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
/**
* This class serves as example code to show how to use the Lucene spatial
@@ -167,7 +167,7 @@ public class SpatialExample extends LuceneTestCase {
//--Match all, order by distance ascending
{
Point pt = ctx.makePoint(60, -50);
- ValueSource valueSource = strategy.makeDistanceValueSource(pt, DistanceUtils.DEG_TO_KM);//the distance (in km)
+ DoubleValuesSource valueSource = strategy.makeDistanceValueSource(pt, DistanceUtils.DEG_TO_KM);//the distance (in km)
Sort distSort = new Sort(valueSource.getSortField(false)).rewrite(indexSearcher);//false=asc dist
TopDocs docs = indexSearcher.search(new MatchAllDocsQuery(), 10, distSort);
assertDocMatchedIds(indexSearcher, docs, 4, 20, 2);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java
index df37d18..199ebcf 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/StrategyTestCase.java
@@ -29,23 +29,21 @@ import java.util.List;
import java.util.Set;
import java.util.logging.Logger;
-import org.locationtech.spatial4j.context.SpatialContext;
-import org.locationtech.spatial4j.shape.Shape;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.function.FunctionQuery;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.CheckHits;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialArgsParser;
import org.apache.lucene.spatial.query.SpatialOperation;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.shape.Shape;
public abstract class StrategyTestCase extends SpatialTestCase {
@@ -212,25 +210,18 @@ public abstract class StrategyTestCase extends SpatialTestCase {
}
/** scores[] are in docId order */
- protected void checkValueSource(ValueSource vs, float scores[], float delta) throws IOException {
- FunctionQuery q = new FunctionQuery(vs);
-
-// //TODO is there any point to this check?
-// int expectedDocs[] = new int[scores.length];//fill with ascending 0....length-1
-// for (int i = 0; i < expectedDocs.length; i++) {
-// expectedDocs[i] = i;
-// }
-// CheckHits.checkHits(random(), q, "", indexSearcher, expectedDocs);
-
- //TopDocs is sorted but we actually don't care about the order
- TopDocs docs = indexSearcher.search(q, 1000);//calculates the score
- for (int i = 0; i < docs.scoreDocs.length; i++) {
- ScoreDoc gotSD = docs.scoreDocs[i];
- float expectedScore = scores[gotSD.doc];
- assertEquals("Not equal for doc "+gotSD.doc, expectedScore, gotSD.score, delta);
+ protected void checkValueSource(DoubleValuesSource vs, float scores[], float delta) throws IOException {
+
+ for (LeafReaderContext ctx : indexSearcher.getTopReaderContext().leaves()) {
+ DoubleValues v = vs.getValues(ctx, null);
+ int count = ctx.reader().maxDoc();
+ for (int i = 0; i < count; i++) {
+ assertTrue(v.advanceExact(i));
+ int doc = i + ctx.docBase;
+ assertEquals("Not equal for doc " + doc, v.doubleValue(), (double) scores[doc], delta);
+ }
}
- CheckHits.checkExplanations(q, "", indexSearcher);
}
protected void testOperation(Shape indexedShape, SpatialOperation operation,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
index fa5900f..26448b6 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dRptTest.java
@@ -29,11 +29,11 @@ import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.query.SpatialOperation;
import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
import org.apache.lucene.spatial3d.geom.GeoBBoxFactory;
+import org.apache.lucene.spatial3d.geom.GeoCircleFactory;
import org.apache.lucene.spatial3d.geom.GeoPathFactory;
import org.apache.lucene.spatial3d.geom.GeoPoint;
import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
import org.apache.lucene.spatial3d.geom.GeoShape;
-import org.apache.lucene.spatial3d.geom.GeoCircleFactory;
import org.apache.lucene.spatial3d.geom.PlanetModel;
import org.junit.Test;
import org.locationtech.spatial4j.context.SpatialContext;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java
index 362809f..af31120 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeRectRelationTestCase.java
@@ -81,7 +81,7 @@ public abstract class Geo3dShapeRectRelationTestCase extends RandomizedShapeTest
return GeoBBoxFactory.makeGeoBBox(planetModel, maxLat, minLat, leftLon, rightLon);
}
- abstract class Geo3dRectIntersectionTestHelper extends RectIntersectionTestHelper<Geo3dShape> {
+ public abstract class Geo3dRectIntersectionTestHelper extends RectIntersectionTestHelper<Geo3dShape> {
public Geo3dRectIntersectionTestHelper(SpatialContext ctx) {
super(ctx);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
index c919eb8..84f90ef 100644
--- a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
+++ b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java
@@ -25,12 +25,13 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.bbox.BBoxOverlapRatioValueSource;
import org.apache.lucene.spatial.query.SpatialArgs;
@@ -257,23 +258,21 @@ public class BBoxStrategy extends SpatialStrategy {
//---------------------------------
/**
- * Provides access to each rectangle per document as a ValueSource in which
- * {@link org.apache.lucene.queries.function.FunctionValues#objectVal(int)} returns a {@link
- * Shape}.
+ * Provides access to each rectangle per document
*/ //TODO raise to SpatialStrategy
- public ValueSource makeShapeValueSource() {
+ public ShapeValuesSource makeShapeValueSource() {
return new BBoxValueSource(this);
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
//TODO if makeShapeValueSource gets lifted to the top; this could become a generic impl.
return new DistanceToShapeValueSource(makeShapeValueSource(), queryPoint, multiplier, ctx);
}
/** Returns a similarity based on {@link BBoxOverlapRatioValueSource}. This is just a
* convenience method. */
- public ValueSource makeOverlapRatioValueSource(Rectangle queryBox, double queryTargetProportion) {
+ public DoubleValuesSource makeOverlapRatioValueSource(Rectangle queryBox, double queryTargetProportion) {
return new BBoxOverlapRatioValueSource(
makeShapeValueSource(), ctx.isGeo(), queryBox, queryTargetProportion, 0.0);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java b/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java
index cd577c7..052020b 100644
--- a/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java
+++ b/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java
@@ -17,16 +17,14 @@
package org.apache.solr.legacy;
import java.io.IOException;
-import java.util.Map;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.Shape;
/**
* A ValueSource in which the indexed Rectangle is returned from
@@ -34,7 +32,7 @@ import org.locationtech.spatial4j.shape.Rectangle;
*
* @lucene.internal
*/
-class BBoxValueSource extends ValueSource {
+class BBoxValueSource extends ShapeValuesSource {
private final BBoxStrategy strategy;
@@ -43,76 +41,34 @@ class BBoxValueSource extends ValueSource {
}
@Override
- public String description() {
+ public String toString() {
return "bboxShape(" + strategy.getFieldName() + ")";
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- LeafReader reader = readerContext.reader();
- final NumericDocValues minX = DocValues.getNumeric(reader, strategy.field_minX);
- final NumericDocValues minY = DocValues.getNumeric(reader, strategy.field_minY);
- final NumericDocValues maxX = DocValues.getNumeric(reader, strategy.field_maxX);
- final NumericDocValues maxY = DocValues.getNumeric(reader, strategy.field_maxY);
+ public ShapeValues getValues(LeafReaderContext readerContext) throws IOException {
+
+ final DoubleValues minX = DoubleValuesSource.fromDoubleField(strategy.field_minX).getValues(readerContext, null);
+ final DoubleValues minY = DoubleValuesSource.fromDoubleField(strategy.field_minY).getValues(readerContext, null);
+ final DoubleValues maxX = DoubleValuesSource.fromDoubleField(strategy.field_maxX).getValues(readerContext, null);
+ final DoubleValues maxY = DoubleValuesSource.fromDoubleField(strategy.field_maxY).getValues(readerContext, null);
//reused
final Rectangle rect = strategy.getSpatialContext().makeRectangle(0,0,0,0);
- return new FunctionValues() {
- private int lastDocID = -1;
-
- private double getDocValue(NumericDocValues values, int doc) throws IOException {
- int curDocID = values.docID();
- if (doc > curDocID) {
- curDocID = values.advance(doc);
- }
- if (doc == curDocID) {
- return Double.longBitsToDouble(values.longValue());
- } else {
- return 0.0;
- }
- }
-
- @Override
- public Object objectVal(int doc) throws IOException {
- if (doc < lastDocID) {
- throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + doc);
- }
- lastDocID = doc;
-
- double minXValue = getDocValue(minX, doc);
- if (minX.docID() != doc) {
- return null;
- } else {
- double minYValue = getDocValue(minY, doc);
- double maxXValue = getDocValue(maxX, doc);
- double maxYValue = getDocValue(maxY, doc);
- rect.reset(minXValue, maxXValue, minYValue, maxYValue);
- return rect;
- }
- }
-
- @Override
- public String strVal(int doc) throws IOException {//TODO support WKT output once Spatial4j does
- Object v = objectVal(doc);
- return v == null ? null : v.toString();
- }
+ return new ShapeValues() {
@Override
- public boolean exists(int doc) throws IOException {
- getDocValue(minX, doc);
- return minX.docID() == doc;
+ public boolean advanceExact(int doc) throws IOException {
+ return minX.advanceExact(doc) && maxX.advanceExact(doc) && minY.advanceExact(doc) && maxY.advanceExact(doc);
}
@Override
- public Explanation explain(int doc) throws IOException {
- return Explanation.match(Float.NaN, toString(doc));
+ public Shape value() throws IOException {
+ rect.reset(minX.doubleValue(), maxX.doubleValue(), minY.doubleValue(), maxY.doubleValue());
+ return rect;
}
- @Override
- public String toString(int doc) throws IOException {
- return description() + '=' + strVal(doc);
- }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/legacy/DistanceValueSource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/legacy/DistanceValueSource.java b/solr/core/src/java/org/apache/solr/legacy/DistanceValueSource.java
index 8685d88..90fd698 100644
--- a/solr/core/src/java/org/apache/solr/legacy/DistanceValueSource.java
+++ b/solr/core/src/java/org/apache/solr/legacy/DistanceValueSource.java
@@ -16,17 +16,13 @@
*/
package org.apache.solr.legacy;
-import org.apache.lucene.index.NumericDocValues;
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
import org.locationtech.spatial4j.distance.DistanceCalculator;
import org.locationtech.spatial4j.shape.Point;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-
-import java.io.IOException;
-import java.util.Map;
/**
* An implementation of the Lucene ValueSource model that returns the distance
@@ -34,11 +30,13 @@ import java.util.Map;
*
* @lucene.internal
*/
-public class DistanceValueSource extends ValueSource {
+public class DistanceValueSource extends DoubleValuesSource {
private PointVectorStrategy strategy;
private final Point from;
private final double multiplier;
+ private final double nullValue;
+
/**
* Constructor.
@@ -47,13 +45,15 @@ public class DistanceValueSource extends ValueSource {
this.strategy = strategy;
this.from = from;
this.multiplier = multiplier;
+ this.nullValue =
+ (strategy.getSpatialContext().isGeo() ? 180 * multiplier : Double.MAX_VALUE);
}
/**
* Returns the ValueSource description.
*/
@Override
- public String description() {
+ public String toString() {
return "DistanceValueSource("+strategy+", "+from+")";
}
@@ -61,55 +61,30 @@ public class DistanceValueSource extends ValueSource {
* Returns the FunctionValues used by the function query.
*/
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- LeafReader reader = readerContext.reader();
-
- final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
- final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());
-
- return new FunctionValues() {
-
- private int lastDocID = -1;
-
- private final Point from = DistanceValueSource.this.from;
- private final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();
- private final double nullValue =
- (strategy.getSpatialContext().isGeo() ? 180 * multiplier : Double.MAX_VALUE);
-
- private double getDocValue(NumericDocValues values, int doc) throws IOException {
- int curDocID = values.docID();
- if (doc > curDocID) {
- curDocID = values.advance(doc);
- }
- if (doc == curDocID) {
- return Double.longBitsToDouble(values.longValue());
- } else {
- return 0.0;
- }
- }
+ public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
- @Override
- public float floatVal(int doc) throws IOException {
- return (float) doubleVal(doc);
- }
+ final DoubleValues ptX = DoubleValuesSource.fromDoubleField(strategy.getFieldNameX()).getValues(readerContext, null);
+ final DoubleValues ptY = DoubleValuesSource.fromDoubleField(strategy.getFieldNameY()).getValues(readerContext, null);
+ final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();
+
+ return DoubleValues.withDefault(new DoubleValues() {
@Override
- public double doubleVal(int doc) throws IOException {
- // make sure it has minX and area
- double x = getDocValue(ptX, doc);
- if (ptX.docID() == doc) {
- double y = getDocValue(ptY, doc);
- assert ptY.docID() == doc;
- return calculator.distance(from, x, y) * multiplier;
- }
- return nullValue;
+ public double doubleValue() throws IOException {
+ return calculator.distance(from, ptX.doubleValue(), ptY.doubleValue()) * multiplier;
}
@Override
- public String toString(int doc) throws IOException {
- return description() + "=" + floatVal(doc);
+ public boolean advanceExact(int doc) throws IOException {
+ return ptX.advanceExact(doc) && ptY.advanceExact(doc);
}
- };
+ }, nullValue);
+
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java b/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
index 3b29a61..da48fcb 100644
--- a/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
+++ b/solr/core/src/java/org/apache/solr/legacy/PointVectorStrategy.java
@@ -23,15 +23,11 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
-import org.apache.solr.legacy.LegacyDoubleField;
-import org.apache.solr.legacy.LegacyFieldType;
-import org.apache.solr.legacy.LegacyNumericRangeQuery;
-import org.apache.solr.legacy.LegacyNumericType;
-import org.apache.lucene.queries.function.FunctionRangeQuery;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.FunctionMatchQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Query;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.query.SpatialArgs;
@@ -218,7 +214,7 @@ public class PointVectorStrategy extends SpatialStrategy {
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
return new DistanceValueSource(this, queryPoint, multiplier);
}
@@ -237,10 +233,11 @@ public class PointVectorStrategy extends SpatialStrategy {
Rectangle bbox = circle.getBoundingBox();
Query approxQuery = makeWithin(bbox);
BooleanQuery.Builder bqBuilder = new BooleanQuery.Builder();
- FunctionRangeQuery vsRangeQuery =
- new FunctionRangeQuery(makeDistanceValueSource(circle.getCenter()), 0.0, circle.getRadius(), true, true);
+ double r = circle.getRadius();
+ FunctionMatchQuery vsMatchQuery = new FunctionMatchQuery(makeDistanceValueSource(circle.getCenter()),
+ v -> 0 <= v && v <= r);
bqBuilder.add(approxQuery, BooleanClause.Occur.FILTER);//should have lowest "cost" value; will drive iteration
- bqBuilder.add(vsRangeQuery, BooleanClause.Occur.FILTER);
+ bqBuilder.add(vsMatchQuery, BooleanClause.Occur.FILTER);
return new ConstantScoreQuery(bqBuilder.build());
} else {
throw new UnsupportedOperationException("Only Rectangles and Circles are currently supported, " +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
index 7b7974b..df54a1a 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java
@@ -20,9 +20,11 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
@@ -35,8 +37,6 @@ import org.apache.solr.response.JSONResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.schema.AbstractSpatialFieldType;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.QParser;
-import org.apache.solr.search.SyntaxError;
import org.locationtech.spatial4j.io.GeoJSONWriter;
import org.locationtech.spatial4j.io.ShapeWriter;
import org.locationtech.spatial4j.io.SupportedFormats;
@@ -87,7 +87,7 @@ public class GeoTransformerFactory extends TransformerFactory
updater.display = display;
updater.display_error = display+"_error";
- ValueSource shapes = null;
+ final ShapeValuesSource shapes;
AbstractSpatialFieldType<?> sdv = (AbstractSpatialFieldType<?>)sf.getType();
SpatialStrategy strategy = sdv.getStrategy(fname);
if(strategy instanceof CompositeSpatialStrategy) {
@@ -98,6 +98,8 @@ public class GeoTransformerFactory extends TransformerFactory
shapes = ((SerializedDVStrategy)strategy)
.makeShapeValueSource();
}
+ else
+ shapes = null;
String writerName = params.get("w", "GeoJSON");
@@ -122,20 +124,24 @@ public class GeoTransformerFactory extends TransformerFactory
// Using ValueSource
if(shapes!=null) {
- // we don't really need the qparser... just so we can reuse valueSource
- QParser parser = new QParser(null,null,params, req) {
+ return new DocTransformer() {
@Override
- public Query parse() throws SyntaxError {
- return new MatchAllDocsQuery();
+ public String getName() {
+ return display;
}
- };
- return new ValueSourceAugmenter(display, parser, shapes) {
@Override
- protected void setValue(SolrDocument doc, Object val) {
- updater.setValue(doc, val);
+ public void transform(SolrDocument doc, int docid, float score) throws IOException {
+ int leafOrd = ReaderUtil.subIndex(docid, context.getSearcher().getTopReaderContext().leaves());
+ LeafReaderContext ctx = context.getSearcher().getTopReaderContext().leaves().get(leafOrd);
+ ShapeValues values = shapes.getValues(ctx);
+ int segmentDoc = docid - ctx.docBase;
+ if (values.advanceExact(segmentDoc)) {
+ updater.setValue(doc, values.value());
+ }
}
};
+
}
// Using the raw stored values
@@ -160,6 +166,7 @@ public class GeoTransformerFactory extends TransformerFactory
}
};
}
+
}
class GeoFieldUpdater {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
index 2106205..2cd1c16 100644
--- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
@@ -34,10 +34,12 @@ import com.google.common.cache.CacheBuilder;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.FunctionScoreQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DoubleValuesSource;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.spatial.SpatialStrategy;
@@ -356,12 +358,12 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
String scoreParam = (localParams == null ? null : localParams.get(SCORE_PARAM));
//We get the valueSource for the score then the filter and combine them.
- ValueSource valueSource = getValueSourceFromSpatialArgs(parser, field, spatialArgs, scoreParam, strategy);
+ DoubleValuesSource valueSource = getValueSourceFromSpatialArgs(parser, field, spatialArgs, scoreParam, strategy);
if (valueSource == null) {
return strategy.makeQuery(spatialArgs); //assumed constant scoring
}
- FunctionQuery functionQuery = new FunctionQuery(valueSource);
+ FunctionScoreQuery functionQuery = new FunctionScoreQuery(new MatchAllDocsQuery(), valueSource);
if (localParams != null && !localParams.getBool(FILTER_PARAM, true))
return functionQuery;
@@ -383,7 +385,7 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
return supportedScoreModes;
}
- protected ValueSource getValueSourceFromSpatialArgs(QParser parser, SchemaField field, SpatialArgs spatialArgs, String score, T strategy) {
+ protected DoubleValuesSource getValueSourceFromSpatialArgs(QParser parser, SchemaField field, SpatialArgs spatialArgs, String score, T strategy) {
if (score == null) {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/schema/BBoxField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/BBoxField.java b/solr/core/src/java/org/apache/solr/schema/BBoxField.java
index 357ccf1..372f61e 100644
--- a/solr/core/src/java/org/apache/solr/schema/BBoxField.java
+++ b/solr/core/src/java/org/apache/solr/schema/BBoxField.java
@@ -23,13 +23,13 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.index.DocValuesType;
-import org.apache.solr.legacy.LegacyFieldType;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.spatial.bbox.BBoxOverlapRatioValueSource;
-import org.apache.solr.legacy.BBoxStrategy;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.util.ShapeAreaValueSource;
import org.apache.solr.common.SolrException;
+import org.apache.solr.legacy.BBoxStrategy;
+import org.apache.solr.legacy.LegacyFieldType;
import org.apache.solr.search.QParser;
import org.locationtech.spatial4j.shape.Rectangle;
@@ -156,7 +156,7 @@ public class BBoxField extends AbstractSpatialFieldType<BBoxStrategy> implements
}
@Override
- protected ValueSource getValueSourceFromSpatialArgs(QParser parser, SchemaField field, SpatialArgs spatialArgs, String scoreParam, BBoxStrategy strategy) {
+ protected DoubleValuesSource getValueSourceFromSpatialArgs(QParser parser, SchemaField field, SpatialArgs spatialArgs, String scoreParam, BBoxStrategy strategy) {
if (scoreParam == null) {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java
index 8ed5484..87f32d9 100644
--- a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java
+++ b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java
@@ -18,16 +18,15 @@
package org.apache.solr.schema;
import java.io.IOException;
-import java.util.Map;
import java.util.Objects;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LatLonDocValuesField;
import org.apache.lucene.document.LatLonPoint;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.LeafFieldComparator;
@@ -177,7 +176,7 @@ public class LatLonPointSpatialField extends AbstractSpatialFieldType implements
}
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
+ public DoubleValuesSource makeDistanceValueSource(Point queryPoint, double multiplier) {
if (!docValues) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
getFieldName() + " must have docValues enabled to support this feature");
@@ -191,7 +190,7 @@ public class LatLonPointSpatialField extends AbstractSpatialFieldType implements
/**
* A {@link ValueSource} based around {@code LatLonDocValuesField#newDistanceSort(String, double, double)}.
*/
- private static class DistanceSortValueSource extends ValueSource {
+ private static class DistanceSortValueSource extends DoubleValuesSource {
private final String fieldName;
private final Point queryPoint;
private final double multiplier;
@@ -218,41 +217,38 @@ public class LatLonPointSpatialField extends AbstractSpatialFieldType implements
}
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- return new DoubleDocValues(this) {
+ public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
+ return new DoubleValues() {
+
@SuppressWarnings("unchecked")
final FieldComparator<Double> comparator =
(FieldComparator<Double>) getSortField(false).getComparator(1, 1);
- final LeafFieldComparator leafComparator = comparator.getLeafComparator(readerContext);
+ final LeafFieldComparator leafComparator = comparator.getLeafComparator(ctx);
final double mult = multiplier; // so it's a local field
- // Since this computation is expensive, it's worth caching it just in case.
- double cacheDoc = -1;
- double cacheVal = Double.POSITIVE_INFINITY;
+ double value = Double.POSITIVE_INFINITY;
@Override
- public double doubleVal(int doc) {
- if (cacheDoc != doc) {
- try {
- leafComparator.copy(0, doc);
- cacheVal = comparator.value(0) * mult;
- cacheDoc = doc;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- return cacheVal;
+ public double doubleValue() throws IOException {
+ return value;
}
@Override
- public boolean exists(int doc) {
- return !Double.isInfinite(doubleVal(doc));
+ public boolean advanceExact(int doc) throws IOException {
+ leafComparator.copy(0, doc);
+ value = comparator.value(0) * mult;
+ return true;
}
};
}
@Override
- public String description() {
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public String toString() {
return "distSort(" + fieldName + ", " + queryPoint + ", mult:" + multiplier + ")";
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
index acd2255..0e123a4 100644
--- a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
+++ b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java
@@ -24,9 +24,8 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.Explanation;
+import org.apache.lucene.spatial.ShapeValues;
+import org.apache.lucene.spatial.ShapeValuesSource;
import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.query.SpatialArgsParser;
@@ -103,24 +102,24 @@ public class RptWithGeometrySpatialField extends AbstractSpatialFieldType<Compos
}
@Override
- public ValueSource makeShapeValueSource() {
+ public ShapeValuesSource makeShapeValueSource() {
return new CachingShapeValuesource(super.makeShapeValueSource(), getFieldName());
}
}
- private static class CachingShapeValuesource extends ValueSource {
+ private static class CachingShapeValuesource extends ShapeValuesSource {
- private final ValueSource targetValueSource;
+ private final ShapeValuesSource targetValueSource;
private final String fieldName;
- private CachingShapeValuesource(ValueSource targetValueSource, String fieldName) {
+ private CachingShapeValuesource(ShapeValuesSource targetValueSource, String fieldName) {
this.targetValueSource = targetValueSource;
this.fieldName = fieldName;
}
@Override
- public String description() {
- return "cache(" + targetValueSource.description() + ")";
+ public String toString() {
+ return "cache(" + targetValueSource.toString() + ")";
}
@Override
@@ -142,10 +141,9 @@ public class RptWithGeometrySpatialField extends AbstractSpatialFieldType<Compos
return result;
}
- @SuppressWarnings("unchecked")
@Override
- public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
- final FunctionValues targetFuncValues = targetValueSource.getValues(context, readerContext);
+ public ShapeValues getValues(LeafReaderContext readerContext) throws IOException {
+ final ShapeValues targetFuncValues = targetValueSource.getValues(readerContext);
// The key is a pair of leaf reader with a docId relative to that reader. The value is a Map from field to Shape.
final SolrCache<PerSegCacheKey,Shape> cache =
SolrRequestInfo.getRequestInfo().getReq().getSearcher().getCache(CACHE_KEY_PREFIX + fieldName);
@@ -153,24 +151,20 @@ public class RptWithGeometrySpatialField extends AbstractSpatialFieldType<Compos
return targetFuncValues; // no caching; no configured cache
}
- return new FunctionValues() {
+ return new ShapeValues() {
int docId = -1;
- Shape shape = null;
- private void setShapeFromDoc(int doc) throws IOException {
- if (docId == doc) {
- return;
- }
- docId = doc;
+ @Override
+ public Shape value() throws IOException {
//lookup in cache
IndexReader.CacheHelper cacheHelper = readerContext.reader().getCoreCacheHelper();
if (cacheHelper == null) {
throw new IllegalStateException("Leaf " + readerContext.reader() + " is not suited for caching");
}
- PerSegCacheKey key = new PerSegCacheKey(cacheHelper.getKey(), doc);
- shape = cache.get(key);
+ PerSegCacheKey key = new PerSegCacheKey(cacheHelper.getKey(), docId);
+ Shape shape = cache.get(key);
if (shape == null) {
- shape = (Shape) targetFuncValues.objectVal(doc);
+ shape = targetFuncValues.value();
if (shape != null) {
cache.put(key, shape);
}
@@ -180,31 +174,15 @@ public class RptWithGeometrySpatialField extends AbstractSpatialFieldType<Compos
((JtsGeometry) shape).index(); // TODO would be nice if some day we didn't have to cast
}
}
- }
-
- // Use the cache for exists & objectVal;
-
- @Override
- public boolean exists(int doc) throws IOException {
- setShapeFromDoc(doc);
- return shape != null;
- }
-
- @Override
- public Object objectVal(int doc) throws IOException {
- setShapeFromDoc(doc);
return shape;
}
@Override
- public Explanation explain(int doc) throws IOException {
- return targetFuncValues.explain(doc);
+ public boolean advanceExact(int doc) throws IOException {
+ this.docId = doc;
+ return targetFuncValues.advanceExact(doc);
}
- @Override
- public String toString(int doc) throws IOException {
- return targetFuncValues.toString(doc);
- }
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java
index 956550c..0829461 100644
--- a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java
+++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java
@@ -20,9 +20,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.locationtech.spatial4j.context.SpatialContext;
-import org.locationtech.spatial4j.distance.DistanceUtils;
-import org.locationtech.spatial4j.shape.Point;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.ConstNumberSource;
import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
@@ -38,6 +35,9 @@ import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.util.DistanceUnits;
import org.apache.solr.util.SpatialUtils;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+import org.locationtech.spatial4j.shape.Point;
/**
* Parses "geodist" creating {@link HaversineConstFunction} or {@link HaversineFunction}
@@ -135,7 +135,7 @@ public class GeoDistValueSourceParser extends ValueSourceParser {
SpatialStrategy strategy = ((SpatialStrategyMultiValueSource) mv2).strategy;
DistanceUnits distanceUnits = ((SpatialStrategyMultiValueSource) mv2).distanceUnits;
Point queryPoint = strategy.getSpatialContext().makePoint(constants[1], constants[0]);
- return strategy.makeDistanceValueSource(queryPoint, distanceUnits.multiplierFromDegreesToThisUnit());
+ return ValueSource.fromDoubleValuesSource(strategy.makeDistanceValueSource(queryPoint, distanceUnits.multiplierFromDegreesToThisUnit()));
}
if (constants != null && other instanceof VectorValueSource) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f2e00ff/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java
index 7ef8f4f..7a8c06a 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial.java
@@ -20,14 +20,10 @@ import java.text.ParseException;
import java.util.Arrays;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.locationtech.spatial4j.context.SpatialContext;
-import org.locationtech.spatial4j.distance.DistanceUtils;
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Rectangle;
-import org.apache.solr.legacy.BBoxStrategy;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.legacy.BBoxStrategy;
import org.apache.solr.schema.BBoxField;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
@@ -35,6 +31,10 @@ import org.apache.solr.util.SpatialUtils;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.locationtech.spatial4j.context.SpatialContext;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Rectangle;
/**
* Test Solr 4's new spatial capabilities from the new Lucene spatial module. Don't thoroughly test it here because
[17/18] lucene-solr:feature/autoscaling: Merge branch 'master' into
feature/autoscaling
Posted by sh...@apache.org.
Merge branch 'master' into feature/autoscaling
# Conflicts:
# solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8d668503
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8d668503
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8d668503
Branch: refs/heads/feature/autoscaling
Commit: 8d668503d8d10690dcd2d5e192bc41b7808fd3a2
Parents: 4239896 b99ee2b
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Jun 28 11:54:23 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Jun 28 11:54:23 2017 +0530
----------------------------------------------------------------------
.../solrj/cloud/autoscaling/TestPolicy.java | 972 +++++++++++++++++++
.../solr/cloud/autoscaling/TestPolicy.java | 972 -------------------
2 files changed, 972 insertions(+), 972 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8d668503/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 0000000,d17e7ce..0fb3d05
mode 000000,100644..100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@@ -1,0 -1,654 +1,972 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.LinkedHashMap;
+ import java.util.List;
+ import java.util.Map;
+
+ import com.google.common.collect.ImmutableList;
+ import org.apache.solr.SolrTestCaseJ4;
+ import org.apache.solr.client.solrj.SolrRequest;
++import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
++import org.apache.solr.common.cloud.Replica;
++import org.apache.solr.common.cloud.ZkStateReader;
+ import org.apache.solr.common.params.CollectionParams;
+ import org.apache.solr.common.params.SolrParams;
+ import org.apache.solr.common.util.Utils;
+ import org.apache.solr.common.util.ValidatingJsonMap;
+
+ import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+ import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+
+ public class TestPolicy extends SolrTestCaseJ4 {
+
+ public static String clusterState = "{'gettingstarted':{" +
+ " 'router':{'name':'compositeId'}," +
+ " 'shards':{" +
+ " 'shard1':{" +
+ " 'range':'80000000-ffffffff'," +
+ " 'replicas':{" +
+ " 'r1':{" +
+ " 'core':r1," +
+ " 'base_url':'http://10.0.0.4:8983/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'," +
+ " 'leader':'true'}," +
+ " 'r2':{" +
+ " 'core':r2," +
+ " 'base_url':'http://10.0.0.4:7574/solr'," +
+ " 'node_name':'node2'," +
+ " 'state':'active'}}}," +
+ " 'shard2':{" +
+ " 'range':'0-7fffffff'," +
+ " 'replicas':{" +
+ " 'r3':{" +
+ " 'core':r3," +
+ " 'base_url':'http://10.0.0.4:8983/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'," +
+ " 'leader':'true'}," +
+ " 'r4':{" +
+ " 'core':r4," +
+ " 'base_url':'http://10.0.0.4:8987/solr'," +
+ " 'node_name':'node4'," +
+ " 'state':'active'}," +
+ " 'r6':{" +
+ " 'core':r6," +
+ " 'base_url':'http://10.0.0.4:8989/solr'," +
+ " 'node_name':'node3'," +
+ " 'state':'active'}," +
+ " 'r5':{" +
+ " 'core':r5," +
- " 'base_url':'http://10.0.0.4:7574/solr'," +
++ " 'base_url':'http://10.0.0.4:8983/solr'," +
+ " 'node_name':'node1'," +
+ " 'state':'active'}}}}}}";
+
- public static Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaDetails(String node, String s) {
++ public static Map<String, Map<String, List<ReplicaInfo>>> getReplicaDetails(String node, String clusterState) {
+ ValidatingJsonMap m = ValidatingJsonMap
- .getDeepCopy((Map) Utils.fromJSONString(s), 6, true);
- Map<String, Map<String, List<Policy.ReplicaInfo>>> result = new LinkedHashMap<>();
++ .getDeepCopy((Map) Utils.fromJSONString(clusterState), 6, true);
++ Map<String, Map<String, List<ReplicaInfo>>> result = new LinkedHashMap<>();
+
+ m.forEach((collName, o) -> {
+ ValidatingJsonMap coll = (ValidatingJsonMap) o;
+ coll.getMap("shards").forEach((shard, o1) -> {
+ ValidatingJsonMap sh = (ValidatingJsonMap) o1;
+ sh.getMap("replicas").forEach((replicaName, o2) -> {
+ ValidatingJsonMap r = (ValidatingJsonMap) o2;
+ String node_name = (String) r.get("node_name");
+ if (!node_name.equals(node)) return;
- Map<String, List<Policy.ReplicaInfo>> shardVsReplicaStats = result.get(collName);
- if (shardVsReplicaStats == null) result.put(collName, shardVsReplicaStats = new HashMap<>());
- List<Policy.ReplicaInfo> replicaInfos = shardVsReplicaStats.get(shard);
- if (replicaInfos == null) shardVsReplicaStats.put(shard, replicaInfos = new ArrayList<>());
- replicaInfos.add(new Policy.ReplicaInfo(replicaName, collName, shard, new HashMap<>()));
++ Map<String, List<ReplicaInfo>> shardVsReplicaStats = result.computeIfAbsent(collName, k -> new HashMap<>());
++ List<ReplicaInfo> replicaInfos = shardVsReplicaStats.computeIfAbsent(shard, k -> new ArrayList<>());
++ replicaInfos.add(new ReplicaInfo(replicaName, collName, shard, Replica.Type.get((String) r.get(ZkStateReader.REPLICA_TYPE)), new HashMap<>()));
+ });
+ });
+ });
+ return result;
+ }
+
+ public void testValidate() {
+ expectError("replica", -1, "must be greater than" );
+ expectError("replica","hello", "not a valid number" );
+ assertEquals( 1l, Clause.validate("replica", "1", true));
+ assertEquals("c", Clause.validate("collection", "c", true));
+ assertEquals( "s", Clause.validate("shard", "s",true));
+ assertEquals( "overseer", Clause.validate("nodeRole", "overseer",true));
+
+ expectError("nodeRole", "wrong","must be one of");
+
+ expectError("sysLoadAvg", "101","must be less than ");
+ expectError("sysLoadAvg", 101,"must be less than ");
+ expectError("sysLoadAvg", "-1","must be greater than");
+ expectError("sysLoadAvg", -1,"must be greater than");
+
+ assertEquals(12.46d,Clause.validate("sysLoadAvg", "12.46",true));
+ assertEquals(12.46,Clause.validate("sysLoadAvg", 12.46d,true));
+
+
+ expectError("ip_1", "300","must be less than ");
+ expectError("ip_1", 300,"must be less than ");
+ expectError("ip_1", "-1","must be greater than");
+ expectError("ip_1", -1,"must be greater than");
+
+ assertEquals(1l,Clause.validate("ip_1", "1",true));
+
+ expectError("heapUsage", "-1","must be greater than");
+ expectError("heapUsage", -1,"must be greater than");
+ assertEquals(69.9d,Clause.validate("heapUsage", "69.9",true));
+ assertEquals(69.9d,Clause.validate("heapUsage", 69.9d,true));
+
+ expectError("port", "70000","must be less than ");
+ expectError("port", 70000,"must be less than ");
+ expectError("port", "0","must be greater than");
+ expectError("port", 0,"must be greater than");
+
+ expectError("cores", "-1","must be greater than");
+
+
+ }
+
+ private static void expectError(String name, Object val, String msg){
+ try {
+ Clause.validate(name, val,true);
+ fail("expected exception containing "+msg);
+ } catch (Exception e) {
+ assertTrue("expected exception containing "+msg,e.getMessage().contains(msg));
+ }
+
+ }
+
+ public void testOperands() {
+ Clause c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:'<2', node:'#ANY'}"));
+ assertFalse(c.replica.isPass(3));
+ assertFalse(c.replica.isPass(2));
+ assertTrue(c.replica.isPass(1));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:'>2', node:'#ANY'}"));
+ assertTrue(c.replica.isPass(3));
+ assertFalse(c.replica.isPass(2));
+ assertFalse(c.replica.isPass(1));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, nodeRole:'!overseer'}"));
+ assertTrue(c.tag.isPass("OVERSEER"));
+ assertFalse(c.tag.isPass("overseer"));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'<12.7'}"));
+ assertTrue(c.tag.isPass("12.6"));
+ assertTrue(c.tag.isPass(12.6d));
+ assertFalse(c.tag.isPass("12.9"));
+ assertFalse(c.tag.isPass(12.9d));
+
+ c = new Clause((Map<String, Object>) Utils.fromJSONString("{replica:0, sysLoadAvg:'>12.7'}"));
+ assertTrue(c.tag.isPass("12.8"));
+ assertTrue(c.tag.isPass(12.8d));
+ assertFalse(c.tag.isPass("12.6"));
+ assertFalse(c.tag.isPass(12.6d));
+ }
+
++ public void testNodeLost() {
++ String dataproviderdata = " {'liveNodes':[" +
++ " '127.0.0.1:65417_solr'," +
++ " '127.0.0.1:65434_solr']," +
++ " 'replicaInfo':{" +
++ " '127.0.0.1:65427_solr':{'testNodeLost':{'shard1':[{'core_node2':{type: NRT}}]}}," +
++ " '127.0.0.1:65417_solr':{'testNodeLost':{'shard1':[{'core_node1':{type: NRT}}]}}," +
++ " '127.0.0.1:65434_solr':{}}," +
++ " 'nodeValues':{" +
++ " '127.0.0.1:65417_solr':{" +
++ " 'node':'127.0.0.1:65417_solr'," +
++ " 'cores':1," +
++ " 'freedisk':884.7097854614258}," +
++ " '127.0.0.1:65434_solr':{" +
++ " 'node':'127.0.0.1:65434_solr'," +
++ " 'cores':0," +
++ " 'freedisk':884.7097854614258}}}";
++ /* String stateJson = "{'testNodeLost':{" +
++ " 'pullReplicas':'0'," +
++ " 'replicationFactor':'2'," +
++ " 'router':{'name':'compositeId'}," +
++ " 'maxShardsPerNode':'1'," +
++ " 'autoAddReplicas':'false'," +
++ " 'nrtReplicas':'2'," +
++ " 'tlogReplicas':'0'," +
++ " 'shards':{'shard1':{" +
++ " 'range':'80000000-7fffffff'," +
++ " 'state':'active'," +
++ " 'replicas':{" +
++ " 'core_node1':{" +
++ " 'core':'testNodeLost_shard1_replica_n1'," +
++ " 'base_url':'http://127.0.0.1:65417/solr'," +
++ " 'node_name':'127.0.0.1:65417_solr'," +
++ " 'state':'active'," +
++ " 'type':'NRT'," +
++ " 'leader':'true'}," +
++ " 'core_node2':{" +
++ " 'core':'testNodeLost_shard1_replica_n2'," +
++ " 'base_url':'http://127.0.0.1:65427/solr'," +
++ " 'node_name':'127.0.0.1:65427_solr'," +
++ " 'state':'down'," +
++ " 'type':'NRT'}}}}}}";*/
++
++ String autoScalingjson = "{" +
++ " 'cluster-policy':[" +
++ " {" +
++ " 'cores':'<10'," +
++ " 'node':'#ANY'}," +
++ " {" +
++ " 'replica':'<2'," +
++ " 'shard':'#EACH'," +
++ " 'node':'#ANY'}," +
++ " {" +
++ " 'nodeRole':'overseer'," +
++ " 'replica':0}]," +
++ " 'cluster-preferences':[" +
++ " {" +
++ " 'minimize':'cores'," +
++ " 'precision':3}," +
++ " {" +
++ " 'maximize':'freedisk'," +
++ " 'precision':100}]}";
++
++ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
++ Policy.Session session = policy.createSession(dataProviderWithData(dataproviderdata));
++ SolrRequest op = session.getSuggester(MOVEREPLICA).hint(Hint.SRC_NODE, "127.0.0.1:65427_solr").getOperation();
++ assertNotNull(op);
++ assertEquals( "127.0.0.1:65434_solr",op.getParams().get("targetNode") );
++ }
++
++ public void testNodeLostMultipleReplica() {
++ String nodeValues = " {" +
++ " 'node4':{" +
++ " 'node':'10.0.0.4:8987_solr'," +
++ " 'cores':1," +
++ " 'freedisk':884.7097854614258}," +
++ " 'node3':{" +
++ " 'node':'10.0.0.4:8989_solr'," +
++ " 'cores':1," +
++ " 'freedisk':884.7097854614258}," +
++ " 'node2':{" +
++ " 'node':'10.0.0.4:7574_solr'," +
++ " 'cores':1," +
++ " 'freedisk':884.7097854614258}," +
++ "}";
++
++ ClusterDataProvider provider = getClusterDataProvider((Map<String, Map>) Utils.fromJSONString(nodeValues), clusterState);
++ Map policies = (Map) Utils.fromJSONString("{" +
++ " 'cluster-preferences': [" +
++ " { 'maximize': 'freedisk', 'precision': 50}," +
++ " { 'minimize': 'cores', 'precision': 50}" +
++ " ]," +
++ " 'cluster-policy': [" +
++ " { 'replica': 0, 'nodeRole': 'overseer'}" +
++ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
++ " ]" +
++ "}");
++ AutoScalingConfig config = new AutoScalingConfig(policies);
++ Policy policy = config.getPolicy();
++ Policy.Session session = policy.createSession(provider);
++ Policy.Suggester suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++
++ SolrRequest operation = suggester.getOperation();
++ assertNotNull(operation);
++ assertEquals("node2", operation.getParams().get("targetNode"));
++
++ session = suggester.getSession();
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++ operation = suggester.getOperation();
++ assertNotNull(operation);
++ assertEquals("node3", operation.getParams().get("targetNode"));
++
++ session = suggester.getSession();
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++ operation = suggester.getOperation();
++ assertNull(operation);
++
++ // lets change the policy such that all replicas that were on node1
++ // can now fit on node2
++ policies = (Map) Utils.fromJSONString("{" +
++ " 'cluster-preferences': [" +
++ " { 'maximize': 'freedisk', 'precision': 50}," +
++ " { 'minimize': 'cores', 'precision': 50}" +
++ " ]," +
++ " 'cluster-policy': [" +
++ " { 'replica': 0, 'nodeRole': 'overseer'}" +
++ " { 'replica': '<3', 'shard': '#EACH', 'node': '#ANY'}," +
++ " ]" +
++ "}");
++ config = new AutoScalingConfig(policies);
++ policy = config.getPolicy();
++ session = policy.createSession(provider);
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++
++ operation = suggester.getOperation();
++ assertNotNull(operation);
++ assertEquals("node2", operation.getParams().get("targetNode"));
++ assertEquals("r3", operation.getParams().get("replica"));
++
++ session = suggester.getSession();
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++ operation = suggester.getOperation();
++ assertNotNull(operation);
++ assertEquals("node2", operation.getParams().get("targetNode"));
++ assertEquals("r5", operation.getParams().get("replica"));
++
++ session = suggester.getSession();
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++ operation = suggester.getOperation();
++ assertEquals("node2", operation.getParams().get("targetNode"));
++ assertEquals("r1", operation.getParams().get("replica"));
++
++ session = suggester.getSession();
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++ operation = suggester.getOperation();
++ assertNull(operation);
++
++ // now lets change the policy such that a node can have 2 shard2 replicas
++ policies = (Map) Utils.fromJSONString("{" +
++ " 'cluster-preferences': [" +
++ " { 'maximize': 'freedisk', 'precision': 50}," +
++ " { 'minimize': 'cores', 'precision': 50}" +
++ " ]," +
++ " 'cluster-policy': [" +
++ " { 'replica': 0, 'nodeRole': 'overseer'}" +
++ " { 'replica': '<2', 'shard': 'shard1', 'node': '#ANY'}," +
++ " { 'replica': '<3', 'shard': 'shard2', 'node': '#ANY'}," +
++ " ]" +
++ "}");
++ config = new AutoScalingConfig(policies);
++ policy = config.getPolicy();
++ session = policy.createSession(provider);
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++
++ operation = suggester.getOperation();
++ assertNotNull(operation);
++ assertEquals("node2", operation.getParams().get("targetNode"));
++ assertEquals("r3", operation.getParams().get("replica"));
++
++ session = suggester.getSession();
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++ operation = suggester.getOperation();
++ assertNotNull(operation);
++ assertEquals("node2", operation.getParams().get("targetNode"));
++ assertEquals("r5", operation.getParams().get("replica"));
++
++ session = suggester.getSession();
++ suggester = session.getSuggester(MOVEREPLICA)
++ .hint(Hint.SRC_NODE, "node1");
++ operation = suggester.getOperation();
++ assertEquals("node3", operation.getParams().get("targetNode"));
++ assertEquals("r1", operation.getParams().get("replica"));
++ }
++
++ private static ClusterDataProvider dataProviderWithData(String data){
++ final Map m = (Map) Utils.fromJSONString(data);
++ Map replicaInfo = (Map) m.get("replicaInfo");
++ replicaInfo.forEach((node, val) -> {
++ Map m1 = (Map) val;
++ m1.forEach((coll, val2) -> {
++ Map m2 = (Map) val2;
++ m2.forEach((shard, val3) -> {
++ List l3 = (List) val3;
++ for (int i = 0; i < l3.size(); i++) {
++ Object o = l3.get(i);
++ Map m3 = (Map) o;
++ l3.set(i, new ReplicaInfo(m3.keySet().iterator().next().toString()
++ ,coll.toString(), shard.toString(), Replica.Type.get((String)m3.get("type")), new HashMap<>()));
++ }
++ });
++
++ });
++
++ });
++ return new ClusterDataProvider(){
++ @Override
++ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
++ return (Map<String, Object>) Utils.getObjectByPath(m,false, Arrays.asList("nodeValues", node));
++ }
++
++ @Override
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ return (Map<String, Map<String, List<ReplicaInfo>>>) Utils.getObjectByPath(m,false, Arrays.asList("replicaInfo", node));
++ }
++
++ @Override
++ public Collection<String> getNodes() {
++ return (Collection<String>) m.get("liveNodes");
++ }
++
++ @Override
++ public String getPolicyNameByCollection(String coll) {
++ return null;
++ }
++ };
++
++
++ }
++
++ public void testPolicyWithReplicaType() {
++ Map policies = (Map) Utils.fromJSONString("{" +
++ " 'cluster-preferences': [" +
++ " { 'maximize': 'freedisk', 'precision': 50}," +
++ " { 'minimize': 'cores', 'precision': 50}" +
++ " ]," +
++ " 'cluster-policy': [" +
++ " { 'replica': 0, 'nodeRole': 'overseer'}" +
++ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
++ " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!ssd', type : TLOG }" +
++ " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!slowdisk' , type : PULL }" +
++ " ]" +
++ "}");
++ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
++ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," +
++ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
++ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
++ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
++ "}");
++ Policy policy = new Policy(policies);
++ Policy.Suggester suggester = policy.createSession(getClusterDataProvider(nodeValues, clusterState))
++ .getSuggester(ADDREPLICA)
++ .hint(Hint.COLL, "newColl")
++ .hint(Hint.REPLICATYPE, Replica.Type.PULL)
++ .hint(Hint.SHARD, "shard1");
++ SolrRequest op = suggester.getOperation();
++ assertNotNull(op);
++ assertEquals(Replica.Type.PULL.name(), op.getParams().get("type"));
++ assertEquals("PULL type node must be in 'slowdisk' node","node1", op.getParams().get("node"));
++
++ suggester = suggester.getSession()
++ .getSuggester(ADDREPLICA)
++ .hint(Hint.COLL, "newColl")
++ .hint(Hint.REPLICATYPE, Replica.Type.PULL)
++ .hint(Hint.SHARD, "shard2");
++ op = suggester.getOperation();
++ assertNotNull(op);
++ assertEquals(Replica.Type.PULL.name(), op.getParams().get("type"));
++ assertEquals("PULL type node must be in 'slowdisk' node","node1", op.getParams().get("node"));
++
++ suggester = suggester.getSession()
++ .getSuggester(ADDREPLICA)
++ .hint(Hint.COLL, "newColl")
++ .hint(Hint.REPLICATYPE, Replica.Type.TLOG)
++ .hint(Hint.SHARD, "shard1");
++ op = suggester.getOperation();
++ assertNotNull(op);
++ assertEquals(Replica.Type.TLOG.name(), op.getParams().get("type"));
++ assertEquals("TLOG type node must be in 'ssd' node","node3", op.getParams().get("node"));
++
++ suggester = suggester.getSession()
++ .getSuggester(ADDREPLICA)
++ .hint(Hint.COLL, "newColl")
++ .hint(Hint.REPLICATYPE, Replica.Type.TLOG)
++ .hint(Hint.SHARD, "shard2");
++ op = suggester.getOperation();
++ assertNotNull(op);
++ assertEquals(Replica.Type.TLOG.name(), op.getParams().get("type"));
++ assertEquals("TLOG type node must be in 'ssd' node","node3", op.getParams().get("node"));
++
++ suggester = suggester.getSession()
++ .getSuggester(ADDREPLICA)
++ .hint(Hint.COLL, "newColl")
++ .hint(Hint.REPLICATYPE, Replica.Type.TLOG)
++ .hint(Hint.SHARD, "shard2");
++ op = suggester.getOperation();
++ assertNull("No node should qualify for this" ,op);
++
++ }
++
+ public void testRow() {
- Row row = new Row("nodex", new Cell[]{new Cell(0, "node", "nodex")}, false, new HashMap<>(), new ArrayList<>());
- Row r1 = row.addReplica("c1", "s1");
- Row r2 = r1.addReplica("c1", "s1");
++ Row row = new Row("nodex", new Cell[]{new Cell(0, "node", "nodex")}, false, new HashMap<>(), new ArrayList<>(), true);
++ Row r1 = row.addReplica("c1", "s1", null);
++ Row r2 = r1.addReplica("c1", "s1",null);
+ assertEquals(1, r1.collectionVsShardVsReplicas.get("c1").get("s1").size());
+ assertEquals(2, r2.collectionVsShardVsReplicas.get("c1").get("s1").size());
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof Policy.ReplicaInfo);
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof Policy.ReplicaInfo);
++ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof ReplicaInfo);
++ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof ReplicaInfo);
+ }
+
+ public void testMerge() {
+
+ Map map = (Map) Utils.fromJSONString("{" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk', 'precision': 50}," +
+ " { 'minimize': 'cores', 'precision': 50}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { 'replica': 0, 'nodeRole': 'overseer'}," +
+ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}" +
+ " ]," +
+ " 'policies': {" +
+ " 'policy1': [" +
+ " { 'replica': '1', 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +
+ " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'}," +
+ " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" +
+ " ]" +
+ " }" +
+ "}");
+ Policy policy = new Policy(map);
+ List<Clause> clauses = Policy.mergePolicies("mycoll", policy.getPolicies().get("policy1"), policy.getClusterPolicy());
+ Collections.sort(clauses);
+ assertEquals(clauses.size(), 4);
+ assertEquals("1", String.valueOf(clauses.get(0).original.get("replica")));
+ assertEquals("0", String.valueOf(clauses.get(1).original.get("replica")));
+ assertEquals("#ANY", clauses.get(3).original.get("shard"));
+ assertEquals("rack1", clauses.get(2).original.get("sysprop.rack"));
+ assertEquals("overseer", clauses.get(1).original.get("nodeRole"));
+ }
+
+ public void testConditionsSort() {
+ String rules = "{" +
+ " 'cluster-policy':[" +
+ " { 'nodeRole':'overseer', replica: 0, 'strict':false}," +
+ " { 'replica':'<1', 'node':'node3', 'shard':'#EACH'}," +
+ " { 'replica':'<2', 'node':'#ANY', 'shard':'#EACH'}," +
+ " { 'replica':1, 'sysprop.rack':'rack1'}]" +
+ " }";
+ Policy p = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
+ List<Clause> clauses = new ArrayList<>(p.getClusterPolicy());
+ Collections.sort(clauses);
+ assertEquals("nodeRole", clauses.get(1).tag.getName());
+ assertEquals("sysprop.rack", clauses.get(0).tag.getName());
+ }
+
+ public void testRules() throws IOException {
+ String rules = "{" +
+ "cluster-policy:[" +
+ "{nodeRole:'overseer',replica : 0 , strict:false}," +
+ "{replica:'<1',node:node3}," +
+ "{replica:'<2',node:'#ANY', shard:'#EACH'}]," +
+ " cluster-preferences:[" +
+ "{minimize:cores , precision:2}," +
+ "{maximize:freedisk, precision:50}, " +
+ "{minimize:heapUsage, precision:1000}]}";
+
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
+ "}");
+
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
+ Policy.Session session;
+ session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
+
+ List<Row> l = session.getSorted();
+ assertEquals("node1", l.get(0).node);
+ assertEquals("node4", l.get(1).node);
+ assertEquals("node3", l.get(2).node);
+ assertEquals("node2", l.get(3).node);
+
+
+ List<Violation> violations = session.getViolations();
+ assertEquals(3, violations.size());
+ assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.getValue())));
+ assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.getName())));
+ assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.getOperand() == Operand.LESS_THAN && "node".equals(violation.getClause().tag.getName()))));
+
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "gettingstarted")
+ .hint(Hint.SHARD, "r1");
+ SolrParams operation = suggester.getOperation().getParams();
+ assertEquals("node2", operation.get("node"));
+
+ nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
+ "node5:{cores:0, freedisk: 895, heapUsage:17834}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
+ "}");
+ session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
+ SolrRequest opReq = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.TARGET_NODE, "node5")
+ .getOperation();
+ assertNotNull(opReq);
+ assertEquals("node5", opReq.getParams().get("targetNode"));
+
+
+ }
+
+ public void testNegativeConditions() {
+ String autoscaleJson = "{" +
+ " 'cluster-policy':[" +
+ " {'replica':'<4','shard':'#EACH','node':'#ANY'}," +
+ " { 'replica': 0, 'sysprop.fs': '!ssd', 'shard': '#EACH'}," +//negative greedy condition
+ " {'nodeRole':'overseer','replica':'0'}]," +
+ " 'cluster-preferences':[" +
+ " {'minimize':'cores', 'precision':3}," +
+ " {'maximize':'freedisk','precision':100}]}";
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "}");
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
+ ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
+ Policy.Session session = policy.createSession(clusterDataProvider);
+ for (int i = 0; i < 3; i++) {
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
+ SolrRequest op = suggester
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1")
+ .getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+ session = suggester.getSession();
+ }
+
+ }
+
+ public void testGreedyConditions() {
+ String autoscaleJson = "{" +
+ " 'cluster-policy':[" +
+ " {'cores':'<10','node':'#ANY'}," +
+ " {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
+ " { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +//greedy condition
+ " {'nodeRole':'overseer','replica':'0'}]," +
+ " 'cluster-preferences':[" +
+ " {'minimize':'cores', 'precision':3}," +
+ " {'maximize':'freedisk','precision':100}]}";
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
+ "}");
+
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
+ ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
+ ClusterDataProvider cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return clusterDataProvider.getNodeValues(node, tags);
+ }
+
+ @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return clusterDataProvider.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return clusterDataProvider.getNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+ };
+ Policy.Session session = policy.createSession(cdp);
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA);
+ SolrRequest op = suggester
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1")
+ .getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+ suggester = suggester
+ .getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals("node3", op.getParams().get("node"));
+
+ suggester = suggester
+ .getSession()
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "shard1");
+ op = suggester.getOperation();
+ assertNotNull(op);
+ assertEquals("node2", op.getParams().get("node"));
+ }
+
+ public void testMoveReplica() {
+ String autoscaleJson = "{" +
+ " 'cluster-policy':[" +
+ " {'cores':'<10','node':'#ANY'}," +
+ " {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
+ " {'nodeRole':'overseer','replica':'0'}]," +
+ " 'cluster-preferences':[" +
+ " {'minimize':'cores', 'precision':3}," +
+ " {'maximize':'freedisk','precision':100}]}";
+
+
+ Map replicaInfoMap = (Map) Utils.fromJSONString("{ '127.0.0.1:60099_solr':{}," +
+ " '127.0.0.1:60089_solr':{'compute_plan_action_test':{'shard1':[" +
+ " {'core_node1':{}}," +
+ " {'core_node2':{}}]}}}");
+ Map m = (Map) Utils.getObjectByPath(replicaInfoMap, false, "127.0.0.1:60089_solr/compute_plan_action_test");
+ m.put("shard1", Arrays.asList(
- new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Collections.emptyMap()),
- new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Collections.emptyMap())
++ new ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Replica.Type.NRT, Collections.emptyMap()),
++ new ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Replica.Type.NRT, Collections.emptyMap())
+ ));
+
+ Map<String, Map<String, Object>> tagsMap = (Map) Utils.fromJSONString("{" +
+ " '127.0.0.1:60099_solr':{" +
+ " 'cores':0," +
+ " 'freedisk':918005641216}," +
+ " '127.0.0.1:60089_solr':{" +
+ " 'cores':2," +
+ " 'freedisk':918005641216}}}");
+
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoscaleJson));
+ Policy.Session session = policy.createSession(new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return tagsMap.get(node);
+ }
+
+ @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return (Map<String, Map<String, List<Policy.ReplicaInfo>>>) replicaInfoMap.get(node);
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ return (Map<String, Map<String, List<ReplicaInfo>>>) replicaInfoMap.get(node);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return replicaInfoMap.keySet();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+ });
+
+ Policy.Suggester suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA)
+ .hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
+ SolrParams op = suggester.getOperation().getParams();
+ assertNotNull(op);
+ session = suggester.getSession();
+ suggester = session.getSuggester(MOVEREPLICA).hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr");
+ op = suggester.getOperation().getParams();
+ assertNotNull(op);
+ }
+
+ public void testOtherTag() {
+ String rules = "{" +
+ "'cluster-preferences':[" +
+ "{'minimize':'cores','precision':2}," +
+ "{'maximize':'freedisk','precision':50}," +
+ "{'minimize':'heapUsage','precision':1000}" +
+ "]," +
+ "'cluster-policy':[" +
+ "{replica:0, 'nodeRole':'overseer','strict':false}," +
+ "{'replica':'<1','node':'node3'}," +
+ "{'replica':'<2','node':'#ANY','shard':'#EACH'}" +
+ "]," +
+ "'policies':{" +
+ "'p1':[" +
+ "{replica:0, 'nodeRole':'overseer','strict':false}," +
+ "{'replica':'<1','node':'node3'}," +
+ "{'replica':'<2','node':'#ANY','shard':'#EACH'}," +
+ "{'replica':'<3','shard':'#EACH','sysprop.rack':'#ANY'}" +
+ "]" +
+ "}" +
+ "}";
+
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, sysprop.rack: rack1}" +
+ "}");
+ Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
+ ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
+ ClusterDataProvider cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return clusterDataProvider.getNodeValues(node, tags);
+ }
+
+ @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return clusterDataProvider.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return clusterDataProvider.getNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return "p1";
+ }
+ };
+ Policy.Session session = policy.createSession(cdp);
+
+ CollectionAdminRequest.AddReplica op = (CollectionAdminRequest.AddReplica) session
+ .getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, "newColl")
+ .hint(Hint.SHARD, "s1").getOperation();
+ assertNotNull(op);
+ assertEquals("node2", op.getNode());
+ }
+
+ private ClusterDataProvider getClusterDataProvider(final Map<String, Map> nodeValues, String clusterState) {
+ return new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return nodeValues.keySet();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+
+ @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return getReplicaDetails(node, clusterState);
+ }
+
+ };
+ }
+ public void testEmptyClusterState(){
+ String autoScaleJson = " {'policies':{'c1':[{" +
+ " 'replica':1," +
+ " 'shard':'#EACH'," +
+ " 'port':'50096'}]}}";
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ " '127.0.0.1:50097_solr':{" +
+ " 'cores':0," +
+ " 'port':'50097'}," +
+ " '127.0.0.1:50096_solr':{" +
+ " 'cores':0," +
+ " 'port':'50096'}}");
+ ClusterDataProvider dataProvider = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return getReplicaDetails(node, clusterState);
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return Arrays.asList( "127.0.0.1:50097_solr", "127.0.0.1:50096_solr");
+ }
+ };
+ Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
+ "newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
+ dataProvider, Collections.singletonMap("newColl", "c1"), Arrays.asList("shard1", "shard2"), 1, null);
+ assertTrue(locations.get("shard1").containsAll(ImmutableList.of("127.0.0.1:50096_solr")));
+ assertTrue(locations.get("shard2").containsAll(ImmutableList.of("127.0.0.1:50096_solr")));
+ }
+
+ public void testMultiReplicaPlacement() {
+ String autoScaleJson = "{" +
+ " 'cluster-preferences': [" +
+ " { maximize : freedisk , precision: 50}," +
+ " { minimize : cores, precision: 2}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " { replica : '0' , 'nodeRole': 'overseer'}," +
+ " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'" +
+ " }" +
+ " ]," +
+ " 'policies': {" +
+ " 'policy1': [" +
+ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" +
+ " ]" +
+ " }" +
+ "}";
+
+
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heap:10480, sysprop.rack:rack3}," +
+ "node2:{cores:4, freedisk: 749, heap:6873, sysprop.fs : ssd, sysprop.rack:rack1}," +
+ "node3:{cores:7, freedisk: 262, heap:7834, sysprop.rack:rack4}," +
+ "node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, sysprop.rack:rack2}" +
+ "}");
+
+ ClusterDataProvider dataProvider = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return getReplicaDetails(node, clusterState);
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return null;
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return Arrays.asList("node1", "node2", "node3", "node4");
+ }
+ };
+ Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
+ "newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
+ dataProvider, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3, null);
+ assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2", "node1", "node3")));
+ assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2", "node1", "node3")));
+
+
+ }
+
+
+ }
[02/18] lucene-solr:feature/autoscaling: SOLR-10272: Fixing precommit
Posted by sh...@apache.org.
SOLR-10272: Fixing precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e4d0bb7d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e4d0bb7d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e4d0bb7d
Branch: refs/heads/feature/autoscaling
Commit: e4d0bb7dc45905b6ab231b940adecd5c5a39eb8e
Parents: 54fc1ee
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Tue Jun 27 17:01:01 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Tue Jun 27 17:01:01 2017 +0530
----------------------------------------------------------------------
.../org/apache/solr/cloud/ZkController.java | 22 +++++++++++++-------
1 file changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4d0bb7d/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 391f29c..f8543db 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -50,6 +50,7 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
+import org.apache.lucene.util.SuppressForbidden;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
@@ -695,7 +696,7 @@ public class ZkController {
* Gets the absolute filesystem path of the _default configset to bootstrap from.
* First tries the sysprop "solr.default.confdir". If not found, tries to find
* the _default dir relative to the sysprop "solr.install.dir".
- * If that fails as well, tries to get the _default from the
+ * If that fails as well (usually for unit tests), tries to get the _default from the
* classpath. Returns null if not found anywhere.
*/
private static String getDefaultConfigDirPath() {
@@ -710,16 +711,23 @@ public class ZkController {
new File(System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) + subPath).exists()) {
configDirPath = new File(System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE) + subPath).getAbsolutePath();
} else { // find "_default" in the classpath. This one is used for tests
- URL classpathUrl = Thread.currentThread().getContextClassLoader().getResource(serverSubPath);
- try {
- if (classpathUrl != null && new File(classpathUrl.toURI()).exists()) {
- configDirPath = new File(classpathUrl.toURI()).getAbsolutePath();
- }
- } catch (URISyntaxException ex) {}
+ configDirPath = getDefaultConfigDirFromClasspath(serverSubPath);
}
return configDirPath;
}
+ @SuppressForbidden(reason = "This will be loaded for unit tests, and hence an exception can be made. "
+ + "In regular server operation, there shouldn't be a situation where we reach this point.")
+ private static String getDefaultConfigDirFromClasspath(String serverSubPath) {
+ URL classpathUrl = Thread.currentThread().getContextClassLoader().getResource(serverSubPath);
+ try {
+ if (classpathUrl != null && new File(classpathUrl.toURI()).exists()) {
+ return new File(classpathUrl.toURI()).getAbsolutePath();
+ }
+ } catch (URISyntaxException ex) {}
+ return null;
+ }
+
private void init(CurrentCoreDescriptorProvider registerOnReconnect) {
try {
[11/18] lucene-solr:feature/autoscaling: Ref Guide: Change
gatherNodes to nodes in the docs
Posted by sh...@apache.org.
Ref Guide: Change gatherNodes to nodes in the docs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ce4a7911
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ce4a7911
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ce4a7911
Branch: refs/heads/feature/autoscaling
Commit: ce4a7911eaa6cb221caaabc74ae2409f88a262f4
Parents: ab319ea
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue Jun 27 12:35:49 2017 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Tue Jun 27 12:36:13 2017 -0400
----------------------------------------------------------------------
solr/solr-ref-guide/src/graph-traversal.adoc | 296 +++++++++++-----------
solr/solr-ref-guide/src/stream-sources.adoc | 4 +-
2 files changed, 150 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce4a7911/solr/solr-ref-guide/src/graph-traversal.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/graph-traversal.adoc b/solr/solr-ref-guide/src/graph-traversal.adoc
index 003a94a..007019b 100644
--- a/solr/solr-ref-guide/src/graph-traversal.adoc
+++ b/solr/solr-ref-guide/src/graph-traversal.adoc
@@ -18,13 +18,13 @@
// specific language governing permissions and limitations
// under the License.
-Graph traversal with streaming expressions uses the `gatherNodes` function to perform a breadth-first graph traversal.
+Graph traversal with streaming expressions uses the `nodes` function to perform a breadth-first graph traversal.
-The `gatherNodes` function can be combined with the `scoreNodes` function to provide recommendations. `gatherNodes` can also be combined with the wider streaming expression library to perform complex operations on gathered node sets.
+The `nodes` function can be combined with the `scoreNodes` function to provide recommendations. `nodes` can also be combined with the wider streaming expression library to perform complex operations on gathered node sets.
-`gatherNodes` traversals are distributed within a SolrCloud collection and can span collections.
+`nodes` traversals are distributed within a SolrCloud collection and can span collections.
-`gatherNodes` is designed for use cases that involve zooming into a neighborhood in the graph and performing precise traversals to gather node sets and aggregations. In these types of use cases `gatherNodes` will often provide sub-second performance. Some sample use cases are provided later in the document.
+`nodes` is designed for use cases that involve zooming into a neighborhood in the graph and performing precise traversals to gather node sets and aggregations. In these types of use cases `nodes` will often provide sub-second performance. Some sample use cases are provided later in the document.
[IMPORTANT]
====
@@ -34,13 +34,13 @@ This document assumes a basic understanding of graph terminology and streaming e
[[GraphTraversal-BasicSyntax]]
== Basic Syntax
-We'll start with the most basic syntax and slowly build up more complexity. The most basic syntax for `gatherNodes` is:
+We'll start with the most basic syntax and slowly build up more complexity. The most basic syntax for `nodes` is:
[source,plain]
----
-gatherNodes(emails,
- walk="johndoe@apache.org->from",
- gather="to")
+nodes(emails,
+ walk="johndoe@apache.org->from",
+ gather="to")
----
Let's break down this simple expression.
@@ -55,20 +55,20 @@ The walk parameter also accepts a list of root node IDs:
[source,plain]
----
-gatherNodes(emails,
- walk="johndoe@apache.org, janesmith@apache.org->from",
- gather="to")
+nodes(emails,
+ walk="johndoe@apache.org, janesmith@apache.org->from",
+ gather="to")
----
-The `gatherNodes` function above finds all the edges with "johndoe@apache.org" or "janesmith@apache.org" in the `from` field and gathers the `to` field.
+The `nodes` function above finds all the edges with "johndoe@apache.org" or "janesmith@apache.org" in the `from` field and gathers the `to` field.
-Like all <<streaming-expressions.adoc#streaming-expressions,Streaming Expressions>>, you can execute a `gatherNodes` expression by sending it to the `/stream` handler. For example:
+Like all <<streaming-expressions.adoc#streaming-expressions,Streaming Expressions>>, you can execute a `nodes` expression by sending it to the `/stream` handler. For example:
[source,bash]
----
-curl --data-urlencode 'expr=gatherNodes(emails,
- walk="johndoe@apache.org, janesmith@apache.org->from",
- gather="to")' http://localhost:8983/solr/emails/stream
+curl --data-urlencode 'expr=nodes(emails,
+ walk="johndoe@apache.org, janesmith@apache.org->from",
+ gather="to")' http://localhost:8983/solr/emails/stream
----
The output of this expression would look like this:
@@ -107,14 +107,14 @@ The output of this expression would look like this:
All of the tuples returned have the `node` field. The `node` field contains the node IDs gathered by the function. The `collection`, `field`, and `level` of the traversal are also included in the output.
-Notice that the level is "1" for each tuple in the example. The root nodes are level 0 (in the example above, the root nodes are "johndoe@apache.org, janesmith@apache.org") By default the `gatherNodes` function emits only the _*leaf nodes*_ of the traversal, which is the outer-most node set. To emit the root nodes you can specify the `scatter` parameter:
+Notice that the level is "1" for each tuple in the example. The root nodes are level 0 (in the example above, the root nodes are "johndoe@apache.org, janesmith@apache.org") By default the `nodes` function emits only the _*leaf nodes*_ of the traversal, which is the outer-most node set. To emit the root nodes you can specify the `scatter` parameter:
[source,plain]
----
-gatherNodes(emails,
- walk="johndoe@apache.org->from",
- gather="to",
- scatter="branches, leaves")
+nodes(emails,
+ walk="johndoe@apache.org->from",
+ gather="to",
+ scatter="branches, leaves")
----
The `scatter` parameter controls whether to emit the _branches_ with the _leaves_. The root nodes are considered "branches" because they are not the outer-most level of the traversal.
@@ -164,14 +164,14 @@ Now the level 0 root node is included in the output.
[[GraphTraversal-Aggregations]]
== Aggregations
-`gatherNodes` also supports aggregations. For example:
+`nodes` also supports aggregations. For example:
[source,plain]
----
-gatherNodes(emails,
- walk="johndoe@apache.org, janesmith@apache.org->from",
- gather="to",
- count(*))
+nodes(emails,
+ walk="johndoe@apache.org, janesmith@apache.org->from",
+ gather="to",
+ count(*))
----
The expression above finds the edges with "\johndoe@apache.org" or "\janesmith@apache.org" in the `from` field and gathers the values from the `to` field. It also aggregates the count for each node ID gathered.
@@ -182,35 +182,35 @@ Edges are uniqued as part of the traversal so the count will *not* reflect the n
The aggregation functions supported are `count(*)`, `sum(field)`, `min(field)`, `max(field)`, and `avg(field)`. The fields being aggregated should be present in the edges collected during the traversal. Later examples (below) will show aggregations can be a powerful tool for providing recommendations and limiting the scope of traversals.
-[[GraphTraversal-NestinggatherNodesfunctions]]
-== Nesting gatherNodes functions
+[[GraphTraversal-Nestingnodesfunctions]]
+== Nesting nodes functions
-The `gatherNodes` function can be nested to traverse deeper into the graph. For example:
+The `nodes` function can be nested to traverse deeper into the graph. For example:
[source,plain]
----
-gatherNodes(emails,
- gatherNodes(emails,
- walk="johndoe@apache.org->from",
- gather="to"),
- walk="node->from",
- gather="to")
+nodes(emails,
+ nodes(emails,
+ walk="johndoe@apache.org->from",
+ gather="to"),
+ walk="node->from",
+ gather="to")
----
-In the example above the outer `gatherNodes` function operates on the node set collected from the inner `gatherNodes` function.
+In the example above the outer `nodes` function operates on the node set collected from the inner `nodes` function.
-Notice that the inner `gatherNodes` function behaves exactly as the examples already discussed. But the `walk` parameter of the outer `gatherNodes` function behaves differently.
+Notice that the inner `nodes` function behaves exactly as the examples already discussed. But the `walk` parameter of the outer `nodes` function behaves differently.
-In the outer `gatherNodes` function the `walk` parameter works with tuples coming from an internal streaming expression. In this scenario the `walk` parameter maps the `node` field to the `from` field. Remember that the node IDs collected from the inner `gatherNodes` expression are placed in the `node` field.
+In the outer `nodes` function the `walk` parameter works with tuples coming from an internal streaming expression. In this scenario the `walk` parameter maps the `node` field to the `from` field. Remember that the node IDs collected from the inner `nodes` expression are placed in the `node` field.
Put more simply, the inner expression gathers all the people that "\johndoe@apache.org" has emailed. We can call this group the "friends of \johndoe@apache.org". The outer expression gathers all the people that the "friends of \johndoe@apache.org" have emailed. This is a basic friends-of-friends traversal.
-This construct of nesting `gatherNodes` functions is the basic technique for doing a controlled traversal through the graph.
+This construct of nesting `nodes` functions is the basic technique for doing a controlled traversal through the graph.
[[GraphTraversal-CycleDetection]]
== Cycle Detection
-The `gatherNodes` function performs cycle detection across the entire traversal. This ensures that nodes that have already been visited are not traversed again. Cycle detection is important for both limiting the size of traversals and gathering accurate aggregations. Without cycle detection the size of the traversal could grow exponentially with each hop in the traversal. With cycle detection only new nodes encountered are traversed.
+The `nodes` function performs cycle detection across the entire traversal. This ensures that nodes that have already been visited are not traversed again. Cycle detection is important for both limiting the size of traversals and gathering accurate aggregations. Without cycle detection the size of the traversal could grow exponentially with each hop in the traversal. With cycle detection only new nodes encountered are traversed.
Cycle detection *does not* cross collection boundaries. This is because internally the collection name is part of the node ID. For example the node ID "\johndoe@apache.org", is really `emails/johndoe@apache.org`. When traversing to another collection "\johndoe@apache.org" will be traversed.
@@ -221,10 +221,10 @@ Each level in the traversal can be filtered with a filter query. For example:
[source,plain]
----
-gatherNodes(emails,
- walk="johndoe@apache.org->from",
- fq="body:(solr rocks)",
- gather="to")
+nodes(emails,
+ walk="johndoe@apache.org->from",
+ fq="body:(solr rocks)",
+ gather="to")
----
In the example above only emails that match the filter query will be included in the traversal. Any Solr query can be included here. So you can do fun things like <<spatial-search.adoc#spatial-search,geospatial queries>>, apply any of the available <<query-syntax-and-parsing.adoc#query-syntax-and-parsing,query parsers>>, or even write custom query parsers to limit the traversal.
@@ -236,10 +236,10 @@ Any streaming expression can be used to provide the root nodes for a traversal.
[source,plain]
----
-gatherNodes(emails,
- search(emails, q="body:(solr rocks)", fl="to", sort="score desc", rows="20")
- walk="to->from",
- gather="to")
+nodes(emails,
+ search(emails, q="body:(solr rocks)", fl="to", sort="score desc", rows="20")
+ walk="to->from",
+ gather="to")
----
The example above provides the root nodes through a search expression. You can also provide arbitrarily complex, nested streaming expressions with joins, etc., to specify the root nodes.
@@ -262,88 +262,88 @@ Look closely at step 2. In large graphs, step 2 can lead to a very large travers
. A large traversal that visit millions of unique nodes is slow and takes a lot of memory because cycle detection is tracked in memory.
. High frequency nodes are also not useful in determining users with similar tastes. The content that fewer people have viewed provides a more precise recommendation.
-The `gatherNodes` function has the `maxDocFreq` param to allow for filtering out high frequency nodes. The sample code below shows steps 1 and 2 of the recommendation:
+The `nodes` function has the `maxDocFreq` param to allow for filtering out high frequency nodes. The sample code below shows steps 1 and 2 of the recommendation:
[source,plain]
----
- gatherNodes(logs,
- search(logs, q="userID:user1", fl="articleID", sort="articleID asc", fq="action:view", qt="/export"),
- walk="articleID->articleID",
- gather="userID",
- fq="action:view",
- maxDocFreq="10000",
- count(*)))
+ nodes(logs,
+ search(logs, q="userID:user1", fl="articleID", sort="articleID asc", fq="action:view", qt="/export"),
+ walk="articleID->articleID",
+ gather="userID",
+ fq="action:view",
+ maxDocFreq="10000",
+ count(*)))
----
-In the example above, the inner search expression searches the `logs` collection and returning all the articles viewed by "user1". The outer `gatherNodes` expression takes all the articles emitted from the inner search expression and finds all the records in the logs collection for those articles. It then gathers and aggregates the users that have read the articles. The `maxDocFreq` parameter limits the articles returned to those that appear in no more then 10,000 log records (per shard). This guards against returning articles that have been viewed by millions of users.
+In the example above, the inner search expression searches the `logs` collection and returning all the articles viewed by "user1". The outer `nodes` expression takes all the articles emitted from the inner search expression and finds all the records in the logs collection for those articles. It then gathers and aggregates the users that have read the articles. The `maxDocFreq` parameter limits the articles returned to those that appear in no more then 10,000 log records (per shard). This guards against returning articles that have been viewed by millions of users.
[[GraphTraversal-TrackingtheTraversal]]
== Tracking the Traversal
-By default the `gatherNodes` function only tracks enough information to do cycle detection. This provides enough information to output the nodes and aggregations in the graph.
+By default the `nodes` function only tracks enough information to do cycle detection. This provides enough information to output the nodes and aggregations in the graph.
-For some use cases, such as graph visualization, we also need to output the edges. Setting `trackTraversal="true"` tells `gatherNodes` to track the connections between nodes, so the edges can be constructed. When `trackTraversal` is enabled a new `ancestors` property will appear with each node. The `ancestors` property contains a list of node IDs that pointed to the node.
+For some use cases, such as graph visualization, we also need to output the edges. Setting `trackTraversal="true"` tells `nodes` to track the connections between nodes, so the edges can be constructed. When `trackTraversal` is enabled a new `ancestors` property will appear with each node. The `ancestors` property contains a list of node IDs that pointed to the node.
-Below is a sample `gatherNodes` expression with `trackTraversal` set to true:
+Below is a sample `nodes` expression with `trackTraversal` set to true:
[source,plain]
----
-gatherNodes(emails,
- gatherNodes(emails,
- walk="johndoe@apache.org->from",
- gather="to",
- trackTraversal="true"),
- walk="node->from",
- trackTraversal="true",
- gather="to")
+nodes(emails,
+ nodes(emails,
+ walk="johndoe@apache.org->from",
+ gather="to",
+ trackTraversal="true"),
+ walk="node->from",
+ trackTraversal="true",
+ gather="to")
----
[[GraphTraversal-Cross-CollectionTraversals]]
== Cross-Collection Traversals
-Nested `gatherNodes` functions can operate on different SolrCloud collections. This allow traversals to "walk" from one collection to another to gather nodes. Cycle detection does not cross collection boundaries, so nodes collected in one collection will be traversed in a different collection. This was done deliberately to support cross-collection traversals. Note that the output from a cross-collection traversal will likely contain duplicate nodes with different collection attributes.
+Nested `nodes` functions can operate on different SolrCloud collections. This allow traversals to "walk" from one collection to another to gather nodes. Cycle detection does not cross collection boundaries, so nodes collected in one collection will be traversed in a different collection. This was done deliberately to support cross-collection traversals. Note that the output from a cross-collection traversal will likely contain duplicate nodes with different collection attributes.
-Below is a sample `gatherNodes` expression that traverses from the "emails" collection to the "logs" collection:
+Below is a sample `nodes` expression that traverses from the "emails" collection to the "logs" collection:
[source,plain]
----
-gatherNodes(logs,
- gatherNodes(emails,
- search(emails, q="body:(solr rocks)", fl="from", sort="score desc", rows="20")
- walk="from->from",
- gather="to",
- scatter="leaves, branches"),
- walk="node->user",
- fq="action:edit",
- gather="contentID")
+nodes(logs,
+ nodes(emails,
+ search(emails, q="body:(solr rocks)", fl="from", sort="score desc", rows="20")
+ walk="from->from",
+ gather="to",
+ scatter="leaves, branches"),
+ walk="node->user",
+ fq="action:edit",
+ gather="contentID")
----
The example above finds all people who sent emails with a body that contains "solr rocks". It then finds all the people these people have emailed. Then it traverses to the logs collection and gathers all the content IDs that these people have edited.
-[[GraphTraversal-CombininggatherNodesWithOtherStreamingExpressions]]
-== Combining gatherNodes With Other Streaming Expressions
+[[GraphTraversal-CombiningnodesWithOtherStreamingExpressions]]
+== Combining nodes With Other Streaming Expressions
-The `gatherNodes` function can act as both a stream source and a stream decorator. The connection with the wider stream expression library provides tremendous power and flexibility when performing graph traversals. Here is an example of using the streaming expression library to intersect two friend networks:
+The `nodes` function can act as both a stream source and a stream decorator. The connection with the wider stream expression library provides tremendous power and flexibility when performing graph traversals. Here is an example of using the streaming expression library to intersect two friend networks:
[source,plain]
----
intersect(on="node",
sort(by="node asc",
- gatherNodes(emails,
- gatherNodes(emails,
- walk="johndoe@apache.org->from",
- gather="to"),
- walk="node->from",
- gather="to",
- scatter="branches,leaves")),
+ nodes(emails,
+ nodes(emails,
+ walk="johndoe@apache.org->from",
+ gather="to"),
+ walk="node->from",
+ gather="to",
+ scatter="branches,leaves")),
sort(by="node asc",
- gatherNodes(emails,
- gatherNodes(emails,
- walk="janedoe@apache.org->from",
- gather="to"),
- walk="node->from",
- gather="to",
- scatter="branches,leaves")))
+ nodes(emails,
+ nodes(emails,
+ walk="janedoe@apache.org->from",
+ gather="to"),
+ walk="node->from",
+ gather="to",
+ scatter="branches,leaves")))
----
The example above gathers two separate friend networks, one rooted with "\johndoe@apache.org" and another rooted with "\janedoe@apache.org". The friend networks are then sorted by the `node` field, and intersected. The resulting node set will be the intersection of the two friend networks.
@@ -362,18 +362,18 @@ Here is the sample syntax:
----
top(n="5",
sort="count(*) desc",
- gatherNodes(baskets,
- random(baskets, q="productID:ABC", fl="basketID", rows="500"),
- walk="basketID->basketID",
- fq="-productID:ABC",
- gather="productID",
- count(*)))
+ nodes(baskets,
+ random(baskets, q="productID:ABC", fl="basketID", rows="500"),
+ walk="basketID->basketID",
+ fq="-productID:ABC",
+ gather="productID",
+ count(*)))
----
Let's break down exactly what this traversal is doing.
. The first expression evaluated is the inner `random` expression, which returns 500 random basketIDs, from the `baskets` collection, that have the `productID` "ABC". The `random` expression is very useful for recommendations because it limits the traversal to a fixed set of baskets, and because it adds the element of surprise into the recommendation. Using the `random` function you can provide fast sample sets from very large graphs.
-. The outer `gatherNodes` expression finds all the records in the `baskets` collection for the basketIDs generated in step 1. It also filters out `productID` "ABC" so it doesn't show up in the results. It then gathers and counts the productID's across these baskets.
+. The outer `nodes` expression finds all the records in the `baskets` collection for the basketIDs generated in step 1. It also filters out `productID` "ABC" so it doesn't show up in the results. It then gathers and counts the productID's across these baskets.
. The outer `top` expression ranks the productIDs emitted in step 2 by the count and selects the top 5.
In a nutshell this expression finds the products that most frequently co-occur with product "ABC" in past shopping baskets.
@@ -388,7 +388,7 @@ Before diving into the syntax of the `scoreNodes` function it's useful to unders
[[GraphTraversal-HowItWorks]]
==== *How It Works*
-The `scoreNodes` function assigns a score to each node emitted by the gatherNodes expression. By default the `scoreNodes` function uses the `count(*)` aggregation, which is the co-occurrence count, as the TF value. The IDF value for each node is fetched from the collection where the node was gathered. Each node is then scored using the TF*IDF formula, which provides a boost to nodes with a lower frequency across all market baskets.
+The `scoreNodes` function assigns a score to each node emitted by the nodes expression. By default the `scoreNodes` function uses the `count(*)` aggregation, which is the co-occurrence count, as the TF value. The IDF value for each node is fetched from the collection where the node was gathered. Each node is then scored using the TF*IDF formula, which provides a boost to nodes with a lower frequency across all market baskets.
Combining the co-occurrence count with the IDF provides a score that shows how important the relationship is between productID ABC and the recommendation candidates.
@@ -403,12 +403,12 @@ top(n="1",
sort="nodeScore desc",
scoreNodes(top(n="50",
sort="count(*) desc",
- gatherNodes(baskets,
- random(baskets, q="productID:ABC", fl="basketID", rows="500"),
- walk="basketID->basketID",
- fq="-productID:ABC",
- gather="productID",
- count(*)))))
+ nodes(baskets,
+ random(baskets, q="productID:ABC", fl="basketID", rows="500"),
+ walk="basketID->basketID",
+ fq="-productID:ABC",
+ gather="productID",
+ count(*)))))
----
This example builds on the earlier example "Calculate market basket co-occurrence".
@@ -428,20 +428,20 @@ Here is the sample syntax:
----
top(n="5",
sort="count(*) desc",
- gatherNodes(logs,
- top(n="30",
- sort="count(*) desc",
- gatherNodes(logs,
- search(logs, q="userID:user1", fl="articleID", sort="articleID asc", fq="action:read", qt="/export"),
- walk="articleID->articleID",
- gather="userID",
- fq="action:read",
- maxDocFreq="10000",
- count(*))),
- walk="node->userID",
- gather="articleID",
- fq="action:read",
- count(*)))
+ nodes(logs,
+ top(n="30",
+ sort="count(*) desc",
+ nodes(logs,
+ search(logs, q="userID:user1", fl="articleID", sort="articleID asc", fq="action:read", qt="/export"),
+ walk="articleID->articleID",
+ gather="userID",
+ fq="action:read",
+ maxDocFreq="10000",
+ count(*))),
+ walk="node->userID",
+ gather="articleID",
+ fq="action:read",
+ count(*)))
----
Let's break down the expression above step-by-step.
@@ -449,11 +449,11 @@ Let's break down the expression above step-by-step.
. The first expression evaluated is the inner `search` expression. This expression searches the `logs` collection for all records matching "user1". This is the user we are making the recommendation for.
+
There is a filter applied to pull back only records where the "action:read". It returns the `articleID` for each record found. In other words, this expression returns all the articles "user1" has read.
-. The inner `gatherNodes` expression operates over the articleIDs returned from step 1. It takes each `articleID` found and searches them against the `articleID` field.
+. The inner `nodes` expression operates over the articleIDs returned from step 1. It takes each `articleID` found and searches them against the `articleID` field.
+
Note that it skips high frequency nodes using the `maxDocFreq` param to filter out articles that appear over 10,000 times in the logs. It gathers userIDs and aggregates the counts for each user. This step finds the users that have read the same articles that "user1" has read and counts how many of the same articles they have read.
. The inner `top` expression ranks the users emitted from step 2. It will emit the top 30 users who have the most overlap with user1's reading list.
-. The outer `gatherNodes` expression gathers the reading list for the users emitted from step 3. It counts the articleIDs that are gathered.
+. The outer `nodes` expression gathers the reading list for the users emitted from step 3. It counts the articleIDs that are gathered.
+
Any article selected in step 1 (user1 reading list), will not appear in this step due to cycle detection. So this step returns the articles read by the users with the most similar readings habits to "user1" that "user1" has not read yet. It also counts the number of times each article has been read across this user group.
. The outer `top` expression takes the top articles emitted from step 4. This is the recommendation.
@@ -467,49 +467,49 @@ The example below illustrates a protein pathway traversal:
[source,plain]
----
-gatherNodes(proteins,
- gatherNodes(proteins,
- walk="NRAS->name",
- gather="interacts"),
- walk="node->name",
- gather="drug")
+nodes(proteins,
+ nodes(proteins,
+ walk="NRAS->name",
+ gather="interacts"),
+ walk="node->name",
+ gather="drug")
----
Let's break down exactly what this traversal is doing.
-. The inner `gatherNodes` expression traverses in the `proteins` collection. It finds all the edges in the graph where the name of the protein is "NRAS". Then it gathers the proteins in the `interacts` field. This gathers all the proteins that "NRAS" interactions with.
-. The outer `gatherNodes` expression also works with the `proteins` collection. It gathers all the drugs that correspond to proteins emitted from step 1.
+. The inner `nodes` expression traverses in the `proteins` collection. It finds all the edges in the graph where the name of the protein is "NRAS". Then it gathers the proteins in the `interacts` field. This gathers all the proteins that "NRAS" interactions with.
+. The outer `nodes` expression also works with the `proteins` collection. It gathers all the drugs that correspond to proteins emitted from step 1.
. Using this stepwise approach you can gather the drugs along the pathway of interactions any number of steps away from the root protein.
[[GraphTraversal-ExportingGraphMLtoSupportGraphVisualization]]
== Exporting GraphML to Support Graph Visualization
-In the examples above, the `gatherNodes` expression was sent to Solr's `/stream` handler like any other streaming expression. This approach outputs the nodes in the same JSON tuple format as other streaming expressions so that it can be treated like any other streaming expression. You can use the `/stream` handler when you need to operate directly on the tuples, such as in the recommendation use cases above.
+In the examples above, the `nodes` expression was sent to Solr's `/stream` handler like any other streaming expression. This approach outputs the nodes in the same JSON tuple format as other streaming expressions so that it can be treated like any other streaming expression. You can use the `/stream` handler when you need to operate directly on the tuples, such as in the recommendation use cases above.
-There are other graph traversal use cases that involve graph visualization. Solr supports these use cases with the introduction of the `/graph` request handler, which takes a `gatherNodes` expression and outputs the results in GraphML.
+There are other graph traversal use cases that involve graph visualization. Solr supports these use cases with the introduction of the `/graph` request handler, which takes a `nodes` expression and outputs the results in GraphML.
-http://graphml.graphdrawing.org/[GraphML] is an XML format supported by graph visualization tools such as https://gephi.org/[Gephi], which is a sophisticated open source tool for statistically analyzing and visualizing graphs. Using a `gatherNodes` expression, parts of a larger graph can be exported in GraphML and then imported into tools like Gephi.
+http://graphml.graphdrawing.org/[GraphML] is an XML format supported by graph visualization tools such as https://gephi.org/[Gephi], which is a sophisticated open source tool for statistically analyzing and visualizing graphs. Using a `nodes` expression, parts of a larger graph can be exported in GraphML and then imported into tools like Gephi.
There are a few things to keep mind when exporting a graph in GraphML:
-. The `/graph` handler can export both the nodes and edges in the graph. By default, it only exports the nodes. To export the edges you must set `trackTraversal="true"` in the `gatherNodes` expression.
-. The `/graph` handler currently accepts an arbitrarily complex streaming expression which includes a `gatherNodes` expression. If the streaming expression doesn't include a `gatherNodes` expression, the `/graph` handler will not properly output GraphML.
-. The `/graph` handler currently accepts a single arbitrarily complex, nested `gatherNodes` expression per request. This means you cannot send in a streaming expression that joins or intersects the node sets from multiple `gatherNodes` expressions. The `/graph` handler does support any level of nesting within a single `gatherNodes` expression. The `/stream` handler does support joining and intersecting node sets, but the `/graph` handler currently does not.
+. The `/graph` handler can export both the nodes and edges in the graph. By default, it only exports the nodes. To export the edges you must set `trackTraversal="true"` in the `nodes` expression.
+. The `/graph` handler currently accepts an arbitrarily complex streaming expression which includes a `nodes` expression. If the streaming expression doesn't include a `nodes` expression, the `/graph` handler will not properly output GraphML.
+. The `/graph` handler currently accepts a single arbitrarily complex, nested `nodes` expression per request. This means you cannot send in a streaming expression that joins or intersects the node sets from multiple `nodes` expressions. The `/graph` handler does support any level of nesting within a single `nodes` expression. The `/stream` handler does support joining and intersecting node sets, but the `/graph` handler currently does not.
[[GraphTraversal-SampleRequest]]
=== Sample Request
[source,bash]
----
-curl --data-urlencode 'expr=gatherNodes(enron_emails,
- gatherNodes(enron_emails,
- walk="kayne.coulter@enron.com->from",
- trackTraversal="true",
- gather="to"),
- walk="node->from",
- scatter="leaves,branches",
+curl --data-urlencode 'expr=nodes(enron_emails,
+ nodes(enron_emails,
+ walk="kayne.coulter@enron.com->from",
trackTraversal="true",
- gather="to")' http://localhost:8983/solr/enron_emails/graph
+ gather="to"),
+ walk="node->from",
+ scatter="leaves,branches",
+ trackTraversal="true",
+ gather="to")' http://localhost:8983/solr/enron_emails/graph
----
[[GraphTraversal-SampleGraphMLOutput]]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce4a7911/solr/solr-ref-guide/src/stream-sources.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-sources.adoc b/solr/solr-ref-guide/src/stream-sources.adoc
index 97db6f2..19a2e55 100644
--- a/solr/solr-ref-guide/src/stream-sources.adoc
+++ b/solr/solr-ref-guide/src/stream-sources.adoc
@@ -213,9 +213,9 @@ features(collection1,
numTerms=250)
----
-== gatherNodes
+== nodes
-The `gatherNodes` function provides breadth-first graph traversal. For details, see the section <<graph-traversal.adoc#graph-traversal,Graph Traversal>>.
+The `nodes` function provides breadth-first graph traversal. For details, see the section <<graph-traversal.adoc#graph-traversal,Graph Traversal>>.
== knn
[03/18] lucene-solr:feature/autoscaling: SOLR-10963: Fix example json
in MultipleAdditiveTreesModel javadocs. (Stefan Langenmaier via Christine
Poerschke)
Posted by sh...@apache.org.
SOLR-10963: Fix example json in MultipleAdditiveTreesModel javadocs. (Stefan Langenmaier via Christine Poerschke)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9c75c808
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9c75c808
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9c75c808
Branch: refs/heads/feature/autoscaling
Commit: 9c75c8082d133a221117dd21cb330e152d76cda4
Parents: e4d0bb7
Author: Christine Poerschke <cp...@apache.org>
Authored: Tue Jun 27 13:22:11 2017 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Tue Jun 27 13:22:11 2017 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../solr/ltr/model/MultipleAdditiveTreesModel.java | 16 ++++++++--------
2 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9c75c808/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7720ef2..7002993 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -464,6 +464,9 @@ Bug Fixes
* SOLR-10824: fix NPE ExactSharedStatsCache, fixing maxdocs skew for terms which are absent at one of shards
when using one of Exact*StatsCache (Mikhail Khludnev)
+* SOLR-10963: Fix example json in MultipleAdditiveTreesModel javadocs.
+ (Stefan Langenmaier via Christine Poerschke)
+
Optimizations
----------------------
* SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9c75c808/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
index 926fab4..278a4f5 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/MultipleAdditiveTreesModel.java
@@ -42,29 +42,29 @@ import org.apache.solr.util.SolrPluginUtils;
"params" : {
"trees" : [
{
- "weight" : 1,
+ "weight" : "1",
"root": {
"feature" : "userTextTitleMatch",
- "threshold" : 0.5,
+ "threshold" : "0.5",
"left" : {
- "value" : -100
+ "value" : "-100"
},
"right" : {
"feature" : "originalScore",
- "threshold" : 10.0,
+ "threshold" : "10.0",
"left" : {
- "value" : 50
+ "value" : "50"
},
"right" : {
- "value" : 75
+ "value" : "75"
}
}
}
},
{
- "weight" : 2,
+ "weight" : "2",
"root" : {
- "value" : -10
+ "value" : "-10"
}
}
]