You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2017/06/28 02:26:00 UTC
[2/2] lucene-solr:master: SOLR-10931: Refactoring the package name
for autoscaling client classes
SOLR-10931: Refactoring the package name for autoscaling client classes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/13a3ae29
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/13a3ae29
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/13a3ae29
Branch: refs/heads/master
Commit: 13a3ae292277d770cb5219142729c14db9744fa5
Parents: ce4a791
Author: Ishan Chattopadhyaya <is...@lucidworks.com>
Authored: Wed Jun 28 07:55:40 2017 +0530
Committer: Ishan Chattopadhyaya <is...@lucidworks.com>
Committed: Wed Jun 28 07:55:40 2017 +0530
----------------------------------------------------------------------
.../src/java/org/apache/solr/cloud/Assign.java | 6 +-
.../cloud/OverseerCollectionMessageHandler.java | 4 +-
.../cloud/autoscaling/AutoScalingHandler.java | 13 +-
.../solr/handler/admin/CollectionsHandler.java | 2 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 69 +++
.../client/solrj/cloud/autoscaling/Cell.java | 69 +++
.../client/solrj/cloud/autoscaling/Clause.java | 472 +++++++++++++++++
.../cloud/autoscaling/ClusterDataProvider.java | 52 ++
.../cloud/autoscaling/MoveReplicaSuggester.java | 83 +++
.../client/solrj/cloud/autoscaling/Operand.java | 123 +++++
.../client/solrj/cloud/autoscaling/Policy.java | 521 +++++++++++++++++++
.../solrj/cloud/autoscaling/PolicyHelper.java | 96 ++++
.../solrj/cloud/autoscaling/Preference.java | 89 ++++
.../client/solrj/cloud/autoscaling/Row.java | 120 +++++
.../solrj/cloud/autoscaling/package-info.java | 23 +
.../solrj/impl/SolrClientDataProvider.java | 4 +-
.../solrj/request/CollectionAdminRequest.java | 2 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 69 ---
.../org/apache/solr/cloud/autoscaling/Cell.java | 57 --
.../apache/solr/cloud/autoscaling/Clause.java | 460 ----------------
.../cloud/autoscaling/ClusterDataProvider.java | 52 --
.../cloud/autoscaling/MoveReplicaSuggester.java | 83 ---
.../apache/solr/cloud/autoscaling/Operand.java | 123 -----
.../apache/solr/cloud/autoscaling/Policy.java | 519 ------------------
.../solr/cloud/autoscaling/PolicyHelper.java | 96 ----
.../solr/cloud/autoscaling/Preference.java | 85 ---
.../org/apache/solr/cloud/autoscaling/Row.java | 116 -----
.../solr/cloud/autoscaling/package-info.java | 23 -
.../apache/solr/common/cloud/DocCollection.java | 2 +-
.../solr/cloud/autoscaling/TestPolicy.java | 23 +-
30 files changed, 1751 insertions(+), 1705 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/cloud/Assign.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Assign.java b/solr/core/src/java/org/apache/solr/cloud/Assign.java
index cdcf415..9f21245 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Assign.java
@@ -29,11 +29,11 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
-import org.apache.solr.cloud.autoscaling.Policy;
-import org.apache.solr.cloud.autoscaling.PolicyHelper;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
@@ -48,7 +48,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.KeeperException;
import static java.util.Collections.singletonMap;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index a055033..c6051d7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -37,12 +37,12 @@ import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.cloud.autoscaling.Policy;
import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.ReplicaAssigner.Position;
@@ -81,7 +81,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 5946bf4..356ce37 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -28,6 +28,11 @@ import java.util.Map;
import com.google.common.collect.ImmutableSet;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.cloud.autoscaling.Cell;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.cloud.autoscaling.Preference;
+import org.apache.solr.client.solrj.cloud.autoscaling.Row;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.common.SolrException;
@@ -139,11 +144,11 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
List<Map<String, Object>> sortedNodes = new ArrayList<>(sorted.size());
for (Row row : sorted) {
Map<String, Object> map = Utils.makeMap("node", row.node);
- for (Cell cell : row.cells) {
+ for (Cell cell : row.getCells()) {
for (Preference clusterPreference : clusterPreferences) {
- Policy.SortParam name = clusterPreference.name;
- if (cell.name.equalsIgnoreCase(name.name())) {
- map.put(name.name(), cell.val);
+ Policy.SortParam name = clusterPreference.getName();
+ if (cell.getName().equalsIgnoreCase(name.name())) {
+ map.put(name.name(), cell.getValue());
break;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 3f2eb5f..122aa4a 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -92,6 +92,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.client.solrj.response.RequestStatusState.COMPLETED;
import static org.apache.solr.client.solrj.response.RequestStatusState.FAILED;
import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND;
@@ -109,7 +110,6 @@ import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_IF_DOW
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.REQUESTID;
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP;
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARD_UNIQUE;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
import static org.apache.solr.common.cloud.DocCollection.RULE;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
new file mode 100644
index 0000000..01149f3
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.List;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+
+class AddReplicaSuggester extends Suggester {
+
+ SolrRequest init() {
+ SolrRequest operation = tryEachNode(true);
+ if (operation == null) operation = tryEachNode(false);
+ return operation;
+ }
+
+ SolrRequest tryEachNode(boolean strict) {
+ String coll = (String) hints.get(Hint.COLL);
+ String shard = (String) hints.get(Hint.SHARD);
+ if (coll == null || shard == null)
+ throw new RuntimeException("add-replica requires 'collection' and 'shard'");
+ //iterate through elements and identify the least loaded
+
+ List<Clause.Violation> leastSeriousViolation = null;
+ Integer targetNodeIndex = null;
+ for (int i = getMatrix().size() - 1; i >= 0; i--) {
+ Row row = getMatrix().get(i);
+ if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
+ Row tmpRow = row.addReplica(coll, shard);
+ tmpRow.violations.clear();
+
+ List<Clause.Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
+ if(!containsNewErrors(errs)) {
+ if(isLessSerious(errs, leastSeriousViolation)){
+ leastSeriousViolation = errs;
+ targetNodeIndex = i;
+ }
+ }
+ }
+
+ if (targetNodeIndex != null) {// there are no rule violations
+ getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
+ return CollectionAdminRequest
+ .addReplicaToShard(coll, shard)
+ .setNode(getMatrix().get(targetNodeIndex).node);
+ }
+
+ return null;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
new file mode 100644
index 0000000..0ac2b36
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Utils;
+
+public class Cell implements MapWriter {
+ final int index;
+ final String name;
+ Object val, approxVal;
+
+ public Cell(int index, String name, Object val) {
+ this.index = index;
+ this.name = name;
+ this.val = val;
+ }
+
+ public Cell(int index, String name, Object val, Object approxVal) {
+ this.index = index;
+ this.name = name;
+ this.val = val;
+ this.approxVal = approxVal;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, val);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(this.toMap(new HashMap<>()));
+ }
+
+ public Cell copy() {
+ return new Cell(index, name, val, approxVal);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Object getValue() {
+ return val;
+ }
+
+ public Object getApproxValue() {
+ return approxVal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
new file mode 100644
index 0000000..ff56627
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+import static java.util.Collections.singletonMap;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
+import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
+import static org.apache.solr.common.params.CoreAdminParams.SHARD;
+
+// a set of conditions in a policy
+public class Clause implements MapWriter, Comparable<Clause> {
+ public Map<String, Object> original;
+ public Condition collection, shard, replica, tag, globalTag;
+
+ boolean strict = true;
+
+ public Clause(Map<String, Object> m) {
+ this.original = m;
+ strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true")));
+ Optional<String> globalTagName = m.keySet().stream().filter(Policy.GLOBAL_ONLY_TAGS::contains).findFirst();
+ if (globalTagName.isPresent()) {
+ globalTag = parse(globalTagName.get(), m);
+ if (m.size() > 2) {
+ throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + Utils.toJSONString(m));
+ }
+ tag = parse(m.keySet().stream()
+ .filter(s -> (!globalTagName.get().equals(s) && !IGNORE_TAGS.contains(s)))
+ .findFirst().get(), m);
+ } else {
+ collection = parse(COLLECTION, m);
+ shard = parse(SHARD, m);
+ if(m.get(REPLICA) == null){
+ throw new RuntimeException(StrUtils.formatString("'replica' is required in {0}", Utils.toJSONString(m)));
+ }
+ this.replica = parse(REPLICA, m);
+ if (replica.op == WILDCARD) throw new RuntimeException("replica val cannot be null" + Utils.toJSONString(m));
+ m.forEach((s, o) -> parseCondition(s, o));
+ }
+ if (tag == null)
+ throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
+
+ }
+
+ public boolean doesOverride(Clause that) {
+ return (collection.equals(that.collection) &&
+ tag.name.equals(that.tag.name));
+
+ }
+
+ public boolean isPerCollectiontag() {
+ return globalTag == null;
+ }
+
+ void parseCondition(String s, Object o) {
+ if (IGNORE_TAGS.contains(s)) return;
+ if (tag != null) {
+ throw new IllegalArgumentException("Only one tag other than collection, shard, replica is possible");
+ }
+ tag = parse(s, singletonMap(s, o));
+ }
+
+ @Override
+ public int compareTo(Clause that) {
+ try {
+ int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
+ if (v != 0) return v;
+ if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
+ v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
+ if (v == 0) {
+ v = Long.compare((Long) this.replica.val, (Long) that.replica.val);
+ v = this.replica.op == LESS_THAN ? v : v * -1;
+ }
+ return v;
+ } else {
+ return 0;
+ }
+ } catch (NullPointerException e) {
+ throw e;
+ }
+ }
+
+ void addTags(List<String> params) {
+ if (globalTag != null && !params.contains(globalTag.name)) params.add(globalTag.name);
+ if (tag != null && !params.contains(tag.name)) params.add(tag.name);
+ }
+
+ public static class Condition {
+ final String name;
+ final Object val;
+ final Operand op;
+
+ Condition(String name, Object val, Operand op) {
+ this.name = name;
+ this.val = val;
+ this.op = op;
+ }
+
+ TestStatus match(Row row) {
+ return op.match(val, row.getVal(name));
+ }
+
+ TestStatus match(Object testVal) {
+ return op.match(this.val, testVal);
+ }
+
+ public boolean isPass(Object inputVal) {
+ return op.match(val, validate(name, inputVal, false)) == PASS;
+ }
+
+ public boolean isPass(Row row) {
+ return op.match(val, row.getVal(name)) == PASS;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that instanceof Condition) {
+ Condition c = (Condition) that;
+ return Objects.equals(c.name, name) && Objects.equals(c.val, val) && c.op == op;
+ }
+ return false;
+ }
+
+ public Integer delta(Object val) {
+ return op.delta(this.val, val);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Object getValue() {
+ return val;
+ }
+
+ public Operand getOperand() {
+ return op;
+ }
+ }
+
+ static Condition parse(String s, Map m) {
+ Object expectedVal = null;
+ Object val = m.get(s);
+ try {
+ String conditionName = s.trim();
+ Operand operand = null;
+ if (val == null) {
+ operand = WILDCARD;
+ expectedVal = Policy.ANY;
+ } else if (val instanceof String) {
+ String strVal = ((String) val).trim();
+ if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
+ else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
+ else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
+ else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
+ else operand = EQUAL;
+ expectedVal = validate(s, strVal.substring(EQUAL == operand || WILDCARD == operand ? 0 : 1), true);
+ } else if (val instanceof Number) {
+ operand = EQUAL;
+ expectedVal = validate(s, val, true);
+ }
+ return new Condition(conditionName, expectedVal, operand);
+
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid tag : " + s + ":" + val, e);
+ }
+ }
+
+ public class Violation implements MapWriter {
+ final String shard, coll, node;
+ final Object actualVal;
+ final Integer delta;//how far is the actual value from the expected value
+ final Object tagKey;
+ private final int hash;
+
+
+ private Violation(String coll, String shard, String node, Object actualVal, Integer delta, Object tagKey) {
+ this.shard = shard;
+ this.coll = coll;
+ this.node = node;
+ this.delta = delta;
+ this.actualVal = actualVal;
+ this.tagKey = tagKey;
+ hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
+ }
+
+ public Clause getClause() {
+ return Clause.this;
+ }
+
+ @Override
+ public int hashCode() {
+ return hash;
+ }
+ //if the delta is lower , this violation is less serious
+ public boolean isLessSerious(Violation that) {
+ return that.delta != null && delta != null &&
+ Math.abs(delta) < Math.abs(that.delta);
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that instanceof Violation) {
+ Violation v = (Violation) that;
+ return Objects.equals(this.shard, v.shard) &&
+ Objects.equals(this.coll, v.coll) &&
+ Objects.equals(this.node, v.node) &&
+ Objects.equals(this.tagKey, v.tagKey)
+ ;
+ }
+ return false;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.putIfNotNull("collection", coll);
+ ew.putIfNotNull("shard", shard);
+ ew.putIfNotNull("node", node);
+ ew.putIfNotNull("tagKey", String.valueOf(tagKey));
+ ew.putIfNotNull("violation", (MapWriter) ew1 -> {
+ ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
+ String.valueOf(actualVal));
+ ew1.putIfNotNull("delta", delta);
+ });
+ ew.put("clause", getClause());
+ }
+ }
+
+
+ public List<Violation> test(List<Row> allRows) {
+ List<Violation> violations = new ArrayList<>();
+ if (isPerCollectiontag()) {
+ Map<String, Map<String, Map<String, AtomicInteger>>> replicaCount = computeReplicaCounts(allRows);
+ for (Map.Entry<String, Map<String, Map<String, AtomicInteger>>> e : replicaCount.entrySet()) {
+ if (!collection.isPass(e.getKey())) continue;
+ for (Map.Entry<String, Map<String, AtomicInteger>> shardVsCount : e.getValue().entrySet()) {
+ if (!shard.isPass(shardVsCount.getKey())) continue;
+ for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
+ if (!replica.isPass(counts.getValue())) {
+ violations.add(new Violation(
+ e.getKey(),
+ shardVsCount.getKey(),
+ tag.name.equals("node") ? counts.getKey() : null,
+ counts.getValue(),
+ replica.delta(counts.getValue()),
+ counts.getKey()
+ ));
+ }
+ }
+ }
+ }
+ } else {
+ for (Row r : allRows) {
+ if (!tag.isPass(r)) {
+ violations.add(new Violation(null, null, r.node, r.getVal(tag.name), tag.delta(r.getVal(tag.name)), null));
+ }
+ }
+ }
+ return violations;
+
+ }
+
+
+ private Map<String, Map<String, Map<String, AtomicInteger>>> computeReplicaCounts(List<Row> allRows) {
+ Map<String, Map<String, Map<String, AtomicInteger>>> collVsShardVsTagVsCount = new HashMap<>();
+ for (Row row : allRows)
+ for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
+ String collectionName = colls.getKey();
+ if (!collection.isPass(collectionName)) continue;
+ collVsShardVsTagVsCount.putIfAbsent(collectionName, new HashMap<>());
+ Map<String, Map<String, AtomicInteger>> collMap = collVsShardVsTagVsCount.get(collectionName);
+ for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
+ String shardName = shards.getKey();
+ if (ANY.equals(shard.val)) shardName = ANY;
+ if (!shard.isPass(shardName)) break;
+ collMap.putIfAbsent(shardName, new HashMap<>());
+ Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
+ Object tagVal = row.getVal(tag.name);
+ tagVsCount.putIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", new AtomicInteger());
+ if (tag.isPass(tagVal)) {
+ tagVsCount.get(String.valueOf(tagVal)).addAndGet(shards.getValue().size());
+ }
+ }
+ }
+ return collVsShardVsTagVsCount;
+ }
+
+ public boolean isStrict() {
+ return strict;
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(original);
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (Map.Entry<String, Object> e : original.entrySet()) ew.put(e.getKey(), e.getValue());
+ }
+
+ enum TestStatus {
+ NOT_APPLICABLE, FAIL, PASS
+ }
+
+ private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict"));
+
+ static class ValidateInfo {
+ final Class type;
+ final Set<String> vals;
+ final Number min;
+ final Number max;
+
+
+ ValidateInfo(Class type, Set<String> vals, Number min, Number max) {
+ this.type = type;
+ this.vals = vals;
+ this.min = min;
+ if(min != null && !type.isInstance(min)) throw new RuntimeException("wrong min value type, expected: " + type.getName() + " actual: " + min.getClass().getName());
+ this.max = max;
+ if(max != null && !type.isInstance(max)) throw new RuntimeException("wrong max value type, expected: " + type.getName() + " actual: " + max.getClass().getName());
+ }
+ }
+
+
+ /**
+ *
+ * @param name name of the condition
+ * @param val value of the condition
+ * @param isRuleVal is this provided in the rule
+ * @return actual validated value
+ */
+ public static Object validate(String name, Object val, boolean isRuleVal) {
+ if (val == null) return null;
+ ValidateInfo info = validatetypes.get(name);
+ if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = validatetypes.get("STRING");
+ if (info == null) throw new RuntimeException("Unknown type :" + name);
+ if (info.type == Double.class) {
+ Double num = parseDouble(name, val);
+ if (isRuleVal) {
+ if (info.min != null)
+ if (Double.compare(num, (Double) info.min) == -1)
+ throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+ if (info.max != null)
+ if (Double.compare(num, (Double) info.max) == 1)
+ throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+ }
+ return num;
+ } else if (info.type == Long.class) {
+ Long num = parseLong(name, val);
+ if (isRuleVal) {
+ if (info.min != null)
+ if (num < info.min.longValue())
+ throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+ if (info.max != null)
+ if (num > info.max.longValue())
+ throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+ }
+ return num;
+ } else if (info.type == String.class) {
+ if (isRuleVal && info.vals != null && !info.vals.contains(val))
+ throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(info.vals, ','));
+ return val;
+ } else {
+ throw new RuntimeException("Invalid type ");
+ }
+ }
+
+ public static Long parseLong(String name, Object val) {
+ if (val == null) return null;
+ if (val instanceof Long) return (Long) val;
+ Number num = null;
+ if (val instanceof String) {
+ try {
+ num = Long.parseLong(((String) val).trim());
+ } catch (NumberFormatException e) {
+ try {
+ num = Double.parseDouble((String) val);
+ } catch (NumberFormatException e1) {
+ throw new RuntimeException(name + ": " + val + "not a valid number", e);
+ }
+ }
+
+ } else if (val instanceof Number) {
+ num = (Number) val;
+ }
+
+ if (num != null) {
+ return num.longValue();
+ }
+ throw new RuntimeException(name + ": " + val + "not a valid number");
+ }
+
+ public static Double parseDouble(String name, Object val) {
+ if (val == null) return null;
+ if (val instanceof Double) return (Double) val;
+ Number num = null;
+ if (val instanceof String) {
+ try {
+ num = Double.parseDouble((String) val);
+ } catch (NumberFormatException e) {
+ throw new RuntimeException(name + ": " + val + "not a valid number", e);
+ }
+
+ } else if (val instanceof Number) {
+ num = (Number) val;
+ }
+
+ if (num != null) {
+ return num.doubleValue();
+ }
+ throw new RuntimeException(name + ": " + val + "not a valid number");
+ }
+
+ private static final Map<String, ValidateInfo> validatetypes = new HashMap<>();
+
+ static {
+ validatetypes.put("collection", new ValidateInfo(String.class, null, null, null));
+ validatetypes.put("shard", new ValidateInfo(String.class, null, null, null));
+ validatetypes.put("replica", new ValidateInfo(Long.class, null, 0L, null));
+ validatetypes.put(ImplicitSnitch.PORT, new ValidateInfo(Long.class, null, 1L, 65535L));
+ validatetypes.put(ImplicitSnitch.DISK, new ValidateInfo(Double.class, null, 0d, Double.MAX_VALUE));
+ validatetypes.put(ImplicitSnitch.NODEROLE, new ValidateInfo(String.class, Collections.singleton("overseer"), null, null));
+ validatetypes.put(ImplicitSnitch.CORES, new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));
+ validatetypes.put(ImplicitSnitch.SYSLOADAVG, new ValidateInfo(Double.class, null, 0d, 100d));
+ validatetypes.put(ImplicitSnitch.HEAPUSAGE, new ValidateInfo(Double.class, null, 0d, null));
+ validatetypes.put("NUMBER", new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));//generic number validation
+ validatetypes.put("STRING", new ValidateInfo(String.class, null, null, null));//generic string validation
+ validatetypes.put("node", new ValidateInfo(String.class, null, null, null));
+ for (String ip : ImplicitSnitch.IP_SNITCHES) validatetypes.put(ip, new ValidateInfo(Long.class, null, 0L, 255L));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
new file mode 100644
index 0000000..e873625
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public interface ClusterDataProvider extends Closeable {
+ /**Get the value of each tag for a given node
+ *
+ * @param node node name
+ * @param tags tag names
+ * @return a map of tag vs value
+ */
+ Map<String, Object> getNodeValues(String node, Collection<String> tags);
+
+ /**
+ * Get the details of each replica in a node. It attempts to fetch as much details about
+ * the replica as mentioned in the keys list. It is not necessary to give al details
+ * <p>
+ * the format is {collection:shard :[{replicadetails}]}
+ */
+ Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
+
+ Collection<String> getNodes();
+
+ /**Get the collection-specific policy
+ */
+ String getPolicyNameByCollection(String coll);
+
+ @Override
+ default void close() throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
new file mode 100644
index 0000000..bf9c284
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.List;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.util.Pair;
+
+public class MoveReplicaSuggester extends Suggester {
+
+ @Override
+ SolrRequest init() {
+ SolrRequest operation = tryEachNode(true);
+ if (operation == null) operation = tryEachNode(false);
+ return operation;
+ }
+
+ SolrRequest tryEachNode(boolean strict) {
+ //iterate through elements and identify the least loaded
+ List<Clause.Violation> leastSeriousViolation = null;
+ Integer targetNodeIndex = null;
+ Integer fromNodeIndex = null;
+ ReplicaInfo fromReplicaInfo = null;
+ for (Pair<ReplicaInfo, Row> fromReplica : getValidReplicas(true, true, -1)) {
+ Row fromRow = fromReplica.second();
+ ReplicaInfo replicaInfo = fromReplica.first();
+ String coll = replicaInfo.collection;
+ String shard = replicaInfo.shard;
+ Pair<Row, ReplicaInfo> pair = fromRow.removeReplica(coll, shard);
+ Row tmpRow = pair.first();
+ if (tmpRow == null) {
+ //no such replica available
+ continue;
+ }
+ tmpRow.violations.clear();
+
+ final int i = getMatrix().indexOf(fromRow);
+ for (int j = getMatrix().size() - 1; j > i; j--) {
+ Row targetRow = getMatrix().get(j);
+ if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
+ targetRow = targetRow.addReplica(coll, shard);
+ targetRow.violations.clear();
+ List<Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getModifiedMatrix(getMatrix(), tmpRow, i), targetRow, j));
+ if (!containsNewErrors(errs) && isLessSerious(errs, leastSeriousViolation)) {
+ leastSeriousViolation = errs;
+ targetNodeIndex = j;
+ fromNodeIndex = i;
+ fromReplicaInfo = replicaInfo;
+ }
+ }
+ }
+ if (targetNodeIndex != null && fromNodeIndex != null) {
+ getMatrix().set(fromNodeIndex, getMatrix().get(fromNodeIndex).removeReplica(fromReplicaInfo.collection, fromReplicaInfo.shard).first());
+ getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(fromReplicaInfo.collection, fromReplicaInfo.shard));
+ return new CollectionAdminRequest.MoveReplica(
+ fromReplicaInfo.collection,
+ fromReplicaInfo.name,
+ getMatrix().get(targetNodeIndex).node);
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
new file mode 100644
index 0000000..e012718
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.FAIL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.NOT_APPLICABLE;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+
+import java.util.Objects;
+
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus;
+
+
+public enum Operand {
+ WILDCARD(ANY, Integer.MAX_VALUE) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return testVal == null ? NOT_APPLICABLE : PASS;
+ }
+
+ },
+ EQUAL("", 0) {
+ @Override
+ public int _delta(int expected, int actual) {
+ return expected - actual;
+ }
+ },
+ NOT_EQUAL("!", 2) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
+ }
+
+ @Override
+ public int _delta(int expected, int actual) {
+ return expected - actual;
+ }
+
+ },
+ GREATER_THAN(">", 1) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ if (testVal == null) return NOT_APPLICABLE;
+ if (ruleVal instanceof Double) {
+ return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == 1 ? PASS : FAIL;
+ }
+ return getLong(testVal) > getLong(ruleVal) ? PASS: FAIL ;
+ }
+
+ @Override
+ protected int _delta(int expected, int actual) {
+ return actual > expected ? 0 : (expected + 1) - actual;
+ }
+ },
+ LESS_THAN("<", 2) {
+ @Override
+ public TestStatus match(Object ruleVal, Object testVal) {
+ if (testVal == null) return NOT_APPLICABLE;
+ if (ruleVal instanceof Double) {
+ return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == -1 ? PASS : FAIL;
+ }
+ return getLong(testVal) < getLong(ruleVal) ? PASS: FAIL ;
+ }
+
+ @Override
+ protected int _delta(int expected, int actual) {
+ return actual < expected ? 0 : (expected ) - actual;
+ }
+
+ };
+ public final String operand;
+ final int priority;
+
+ Operand(String val, int priority) {
+ this.operand = val;
+ this.priority = priority;
+ }
+
+ public String toStr(Object expectedVal) {
+ return operand + expectedVal.toString();
+ }
+
+ public TestStatus match(Object ruleVal, Object testVal) {
+ return Objects.equals(ruleVal, testVal) ? PASS : FAIL;
+ }
+
+ Long getLong(Object o) {
+ if (o instanceof Long) return (Long) o;
+ if(o instanceof Number ) return ((Number) o).longValue();
+ return Long.parseLong(String.valueOf(o));
+
+ }
+
+ public Integer delta(Object expected, Object actual) {
+ try {
+ Integer expectedInt = Integer.parseInt(String.valueOf(expected));
+ Integer actualInt = Integer.parseInt(String.valueOf(actual));
+ return _delta(expectedInt, actualInt);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ protected int _delta(int expected, int actual) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
new file mode 100644
index 0000000..ccb0dee
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.stream.Collectors.toList;
+
+/*The class that reads, parses and applies policies specified in
+ * autoscaling.json
+ *
+ * Create one instance of this class per unique autoscaling.json.
+ * This is immutable and is thread-safe
+ *
+ * Create a fresh new session for each use
+ *
+ */
+public class Policy implements MapWriter {
+ public static final String POLICY = "policy";
+ public static final String EACH = "#EACH";
+ public static final String ANY = "#ANY";
+ public static final String CLUSTER_POLICY = "cluster-policy";
+ public static final String CLUSTER_PREFERENCE = "cluster-preferences";
+ public static final Set<String> GLOBAL_ONLY_TAGS = Collections.singleton("cores");
+ final Map<String, List<Clause>> policies = new HashMap<>();
+ final List<Clause> clusterPolicy;
+ final List<Preference> clusterPreferences;
+ final List<String> params;
+
+
+ public Policy(Map<String, Object> jsonMap) {
+
+ clusterPreferences = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_PREFERENCE, emptyList())).stream()
+ .map(Preference::new)
+ .collect(toList());
+ for (int i = 0; i < clusterPreferences.size() - 1; i++) {
+ Preference preference = clusterPreferences.get(i);
+ preference.next = clusterPreferences.get(i + 1);
+ }
+ if (clusterPreferences.isEmpty()) {
+ clusterPreferences.add(new Preference((Map<String, Object>) Utils.fromJSONString("{minimize : cores, precision:1}")));
+ }
+ SortedSet<String> paramsOfInterest = new TreeSet<>();
+ for (Preference preference : clusterPreferences) {
+ if (paramsOfInterest.contains(preference.name.name())) {
+ throw new RuntimeException(preference.name + " is repeated");
+ }
+ paramsOfInterest.add(preference.name.toString());
+ }
+ this.params = new ArrayList<>(paramsOfInterest);
+
+ clusterPolicy = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_POLICY, emptyList())).stream()
+ .map(Clause::new)
+ .filter(clause -> {
+ clause.addTags(params);
+ return true;
+ })
+ .collect(Collectors.toList());
+
+ ((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault("policies", emptyMap())).forEach((s, l1) ->
+ this.policies.put(s, l1.stream()
+ .map(Clause::new)
+ .filter(clause -> {
+ if (!clause.isPerCollectiontag())
+ throw new RuntimeException(clause.globalTag.name + " is only allowed in 'cluster-policy'");
+ clause.addTags(params);
+ return true;
+ })
+ .sorted()
+ .collect(toList())));
+ }
+
+ public List<Clause> getClusterPolicy() {
+ return clusterPolicy;
+ }
+
+ public List<Preference> getClusterPreferences() {
+ return clusterPreferences;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ if (!policies.isEmpty()) {
+ ew.put("policies", (MapWriter) ew1 -> {
+ for (Map.Entry<String, List<Clause>> e : policies.entrySet()) {
+ ew1.put(e.getKey(), e.getValue());
+ }
+ });
+ }
+ if (!clusterPreferences.isEmpty()) {
+ ew.put("preferences", (IteratorWriter) iw -> {
+ for (Preference p : clusterPreferences) iw.add(p);
+ });
+ }
+
+ }
+
+ /*This stores the logical state of the system, given a policy and
+ * a cluster state.
+ *
+ */
+ public class Session implements MapWriter {
+ final List<String> nodes;
+ final ClusterDataProvider dataProvider;
+ final List<Row> matrix;
+ Set<String> collections = new HashSet<>();
+ List<Clause> expandedClauses;
+ List<Violation> violations = new ArrayList<>();
+
+ private Session(List<String> nodes, ClusterDataProvider dataProvider,
+ List<Row> matrix, List<Clause> expandedClauses) {
+ this.nodes = nodes;
+ this.dataProvider = dataProvider;
+ this.matrix = matrix;
+ this.expandedClauses = expandedClauses;
+ }
+
+ Session(ClusterDataProvider dataProvider) {
+ this.nodes = new ArrayList<>(dataProvider.getNodes());
+ this.dataProvider = dataProvider;
+ for (String node : nodes) {
+ collections.addAll(dataProvider.getReplicaInfo(node, Collections.emptyList()).keySet());
+ }
+
+ expandedClauses = clusterPolicy.stream()
+ .filter(clause -> !clause.isPerCollectiontag())
+ .collect(Collectors.toList());
+
+ for (String c : collections) {
+ addClausesForCollection(dataProvider, c);
+ }
+
+ Collections.sort(expandedClauses);
+
+ matrix = new ArrayList<>(nodes.size());
+ for (String node : nodes) matrix.add(new Row(node, params, dataProvider));
+ applyRules();
+ }
+
+ private void addClausesForCollection(ClusterDataProvider dataProvider, String c) {
+ String p = dataProvider.getPolicyNameByCollection(c);
+ if (p != null) {
+ List<Clause> perCollPolicy = policies.get(p);
+ if (perCollPolicy == null)
+ throw new RuntimeException(StrUtils.formatString("Policy for collection {0} is {1} . It does not exist", c, p));
+ }
+ expandedClauses.addAll(mergePolicies(c, policies.getOrDefault(p, emptyList()), clusterPolicy));
+ }
+
+ Session copy() {
+ return new Session(nodes, dataProvider, getMatrixCopy(), expandedClauses);
+ }
+
+ List<Row> getMatrixCopy() {
+ return matrix.stream()
+ .map(Row::copy)
+ .collect(Collectors.toList());
+ }
+
+ Policy getPolicy() {
+ return Policy.this;
+
+ }
+
+ /**
+ * Apply the preferences and conditions
+ */
+ private void applyRules() {
+ if (!clusterPreferences.isEmpty()) {
+ //this is to set the approximate value according to the precision
+ ArrayList<Row> tmpMatrix = new ArrayList<>(matrix);
+ for (Preference p : clusterPreferences) {
+ Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false));
+ p.setApproxVal(tmpMatrix);
+ }
+ //approximate values are set now. Let's do recursive sorting
+ Collections.sort(matrix, (Row r1, Row r2) -> {
+ int result = clusterPreferences.get(0).compare(r1, r2, true);
+ if (result == 0) result = clusterPreferences.get(0).compare(r1, r2, false);
+ return result;
+ });
+ }
+
+ for (Clause clause : expandedClauses) {
+ List<Violation> errs = clause.test(matrix);
+ violations.addAll(errs);
+ }
+ }
+
+ public List<Violation> getViolations() {
+ return violations;
+ }
+
+ public Suggester getSuggester(CollectionAction action) {
+ Suggester op = ops.get(action).get();
+ if (op == null) throw new UnsupportedOperationException(action.toString() + "is not supported");
+ op._init(this);
+ return op;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (int i = 0; i < matrix.size(); i++) {
+ Row row = matrix.get(i);
+ ew.put(row.node, row);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(toMap(new LinkedHashMap<>()));
+ }
+
+ public List<Row> getSorted() {
+ return Collections.unmodifiableList(matrix);
+ }
+ }
+
+
+ public Session createSession(ClusterDataProvider dataProvider) {
+ return new Session(dataProvider);
+ }
+
+ public enum SortParam {
+ freedisk(0, Integer.MAX_VALUE), cores(0, Integer.MAX_VALUE), heapUsage(0, Integer.MAX_VALUE), sysLoadAvg(0, 100);
+
+ public final int min,max;
+
+ SortParam(int min, int max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ static SortParam get(String m) {
+ for (SortParam p : values()) if (p.name().equals(m)) return p;
+ throw new RuntimeException(StrUtils.formatString("Invalid sort {0} Sort must be on one of these {1}", m, Arrays.asList(values())));
+ }
+ }
+
+ enum Sort {
+ maximize(1), minimize(-1);
+ final int sortval;
+
+ Sort(int i) {
+ sortval = i;
+ }
+
+ static Sort get(Map<String, Object> m) {
+ if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) {
+ throw new RuntimeException("Cannot have both 'maximize' and 'minimize'");
+ }
+ if (m.containsKey(maximize.name())) return maximize;
+ if (m.containsKey(minimize.name())) return minimize;
+ throw new RuntimeException("must have either 'maximize' or 'minimize'");
+ }
+ }
+
+
+ public static class ReplicaInfo implements MapWriter {
+ final String name;
+ String core, collection, shard;
+ Map<String, Object> variables;
+
+ public ReplicaInfo(String name, String coll, String shard, Map<String, Object> vals) {
+ this.name = name;
+ this.variables = vals;
+ this.collection = coll;
+ this.shard = shard;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, variables);
+ }
+
+ public String getCore() {
+ return core;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public String getShard() {
+ return shard;
+ }
+ }
+
+
+ /* A suggester is capable of suggesting a collection operation
+ * given a particular session. Before it suggests a new operation,
+ * it ensures that ,
+ * a) load is reduced on the most loaded node
+ * b) it causes no new violations
+ *
+ */
+ public static abstract class Suggester {
+ protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
+ Policy.Session session;
+ SolrRequest operation;
+ protected List<Violation> originalViolations = new ArrayList<>();
+ private boolean isInitialized = false;
+
+ private void _init(Session session) {
+ this.session = session.copy();
+ }
+
+ public Suggester hint(Hint hint, Object value) {
+ if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+ ((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).add(value);
+ } else {
+ hints.put(hint, value);
+ }
+ return this;
+ }
+
+ abstract SolrRequest init();
+
+
+ public SolrRequest getOperation() {
+ if (!isInitialized) {
+ String coll = (String) hints.get(Hint.COLL);
+ String shard = (String) hints.get(Hint.SHARD);
+ // if this is not a known collection from the existing clusterstate,
+ // then add it
+ if (session.matrix.stream().noneMatch(row -> row.collectionVsShardVsReplicas.containsKey(coll))) {
+ session.addClausesForCollection(session.dataProvider, coll);
+ Collections.sort(session.expandedClauses);
+ }
+ if (coll != null) {
+ for (Row row : session.matrix) {
+ if (!row.collectionVsShardVsReplicas.containsKey(coll)) row.collectionVsShardVsReplicas.put(coll, new HashMap<>());
+ if (shard != null) {
+ Map<String, List<ReplicaInfo>> shardInfo = row.collectionVsShardVsReplicas.get(coll);
+ if (!shardInfo.containsKey(shard)) shardInfo.put(shard, new ArrayList<>());
+ }
+ }
+ }
+ session.applyRules();
+ originalViolations.addAll(session.getViolations());
+ this.operation = init();
+ isInitialized = true;
+ }
+ return operation;
+ }
+
+ public Session getSession() {
+ return session;
+ }
+
+ List<Row> getMatrix() {
+ return session.matrix;
+
+ }
+
+ //check if the fresh set of violations is less serious than the last set of violations
+ boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
+ if (old == null || fresh.size() < old.size()) return true;
+ if (fresh.size() == old.size()) {
+ for (int i = 0; i < fresh.size(); i++) {
+ Violation freshViolation = fresh.get(i);
+ Violation oldViolation = null;
+ for (Violation v : old) {
+ if (v.equals(freshViolation)) oldViolation = v;
+ }
+ if (oldViolation != null && freshViolation.isLessSerious(oldViolation)) return true;
+ }
+ }
+ return false;
+ }
+
+ boolean containsNewErrors(List<Violation> violations) {
+ for (Violation v : violations) {
+ int idx = originalViolations.indexOf(v);
+ if (idx < 0 || originalViolations.get(idx).isLessSerious(v)) return true;
+ }
+ return false;
+ }
+
+ List<Pair<ReplicaInfo, Row>> getValidReplicas(boolean sortDesc, boolean isSource, int until) {
+ List<Pair<Policy.ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
+
+ if (sortDesc) {
+ if (until == -1) until = getMatrix().size();
+ for (int i = 0; i < until; i++) addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+ } else {
+ if (until == -1) until = 0;
+ for (int i = getMatrix().size() - 1; i >= until; i--)
+ addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+ }
+ return allPossibleReplicas;
+ }
+
+ void addReplicaToList(Row r, boolean isSource, List<Pair<Policy.ReplicaInfo, Row>> replicaList) {
+ if (!isAllowed(r.node, isSource ? Hint.SRC_NODE : Hint.TARGET_NODE)) return;
+ for (Map.Entry<String, Map<String, List<Policy.ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
+ if (!isAllowed(e.getKey(), Hint.COLL)) continue;
+ for (Map.Entry<String, List<Policy.ReplicaInfo>> shard : e.getValue().entrySet()) {
+ if (!isAllowed(e.getKey(), Hint.SHARD)) continue;
+ replicaList.add(new Pair<>(shard.getValue().get(0), r));
+ }
+ }
+ }
+
+ protected List<Violation> testChangedMatrix(boolean strict, List<Row> rows) {
+ List<Violation> errors = new ArrayList<>();
+ for (Clause clause : session.expandedClauses) {
+ if (strict || clause.strict) {
+ List<Violation> errs = clause.test(rows);
+ if (!errs.isEmpty()) {
+ errors.addAll(errs);
+ }
+ }
+ }
+ return errors;
+ }
+
+ ArrayList<Row> getModifiedMatrix(List<Row> matrix, Row tmpRow, int i) {
+ ArrayList<Row> copy = new ArrayList<>(matrix);
+ copy.set(i, tmpRow);
+ return copy;
+ }
+
+ protected boolean isAllowed(Object v, Hint hint) {
+ Object hintVal = hints.get(hint);
+ if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+ Set set = (Set) hintVal;
+ return set == null || set.contains(v);
+ } else {
+ return hintVal == null || Objects.equals(v, hintVal);
+ }
+ }
+
+ public enum Hint {
+ COLL, SHARD, SRC_NODE, TARGET_NODE
+ }
+
+
+ }
+
+ public static List<Clause> mergePolicies(String coll,
+ List<Clause> collPolicy,
+ List<Clause> globalPolicy) {
+
+ List<Clause> merged = insertColl(coll, collPolicy);
+ List<Clause> global = insertColl(coll, globalPolicy);
+ merged.addAll(global.stream()
+ .filter(clusterPolicyClause -> merged.stream().noneMatch(perCollPolicy -> perCollPolicy.doesOverride(clusterPolicyClause)))
+ .collect(Collectors.toList()));
+ return merged;
+ }
+
+ /**
+ * Insert the collection name into the clauses where collection is not specified
+ */
+ static List<Clause> insertColl(String coll, Collection<Clause> conditions) {
+ return conditions.stream()
+ .filter(Clause::isPerCollectiontag)
+ .map(clause -> {
+ Map<String, Object> copy = new LinkedHashMap<>(clause.original);
+ if (!copy.containsKey("collection")) copy.put("collection", coll);
+ return new Clause(copy);
+ })
+ .filter(it -> (it.collection.isPass(coll)))
+ .collect(Collectors.toList());
+
+ }
+
+ private static final Map<CollectionAction, Supplier<Suggester>> ops = new HashMap<>();
+
+ static {
+ ops.put(CollectionAction.ADDREPLICA, () -> new AddReplicaSuggester());
+ ops.put(CollectionAction.MOVEREPLICA, () -> new MoveReplicaSuggester());
+ }
+
+ public Map<String, List<Clause>> getPolicies() {
+ return policies;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
new file mode 100644
index 0000000..deea175
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.Utils;
+
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+
+public class PolicyHelper {
+ public static Map<String, List<String>> getReplicaLocations(String collName, Map<String, Object> autoScalingJson,
+ ClusterDataProvider cdp,
+ Map<String, String> optionalPolicyMapping,
+ List<String> shardNames,
+ int repFactor,
+ List<String> nodesList) {
+ Map<String, List<String>> positionMapping = new HashMap<>();
+ for (String shardName : shardNames) positionMapping.put(shardName, new ArrayList<>(repFactor));
+ if (optionalPolicyMapping != null) {
+ final ClusterDataProvider delegate = cdp;
+ cdp = new ClusterDataProvider() {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ return delegate.getNodeValues(node, tags);
+ }
+
+ @Override
+ public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return delegate.getReplicaInfo(node, keys);
+ }
+
+ @Override
+ public Collection<String> getNodes() {
+ return delegate.getNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ return optionalPolicyMapping.containsKey(coll) ?
+ optionalPolicyMapping.get(coll) :
+ delegate.getPolicyNameByCollection(coll);
+ }
+ };
+
+ }
+
+
+ Policy policy = new Policy(autoScalingJson);
+ Policy.Session session = policy.createSession(cdp);
+ for (String shardName : shardNames) {
+ for (int i = 0; i < repFactor; i++) {
+ Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
+ .hint(Hint.COLL, collName)
+ .hint(Hint.SHARD, shardName);
+ if (nodesList != null) {
+ for (String nodeName : nodesList) {
+ suggester = suggester.hint(Hint.TARGET_NODE, nodeName);
+ }
+ }
+ SolrRequest op = suggester.getOperation();
+ if (op == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(Utils.getDeepCopy(session.expandedClauses, 4, true)));
+ }
+ session = suggester.getSession();
+ positionMapping.get(shardName).add(op.getParams().get(CoreAdminParams.NODE));
+ }
+ }
+
+ return positionMapping;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
new file mode 100644
index 0000000..bb45628
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+public class Preference implements MapWriter {
+ final Policy.SortParam name;
+ Integer precision;
+ final Policy.Sort sort;
+ Preference next;
+ public int idx;
+ private final Map original;
+
+ Preference(Map<String, Object> m) {
+ this.original = Utils.getDeepCopy(m,3);
+ sort = Policy.Sort.get(m);
+ name = Policy.SortParam.get(m.get(sort.name()).toString());
+ Object p = m.getOrDefault("precision", 0);
+ precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
+ if (precision < 0) {
+ throw new RuntimeException("precision must be a positive value ");
+ }
+ if(precision< name.min || precision> name.max){
+ throw new RuntimeException(StrUtils.formatString("invalid precision value {0} must lie between {1} and {1}",
+ precision, name.min, name.max ) );
+ }
+
+ }
+
+ // there are 2 modes of compare.
+ // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
+ // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
+ int compare(Row r1, Row r2, boolean useApprox) {
+ Object o1 = useApprox ? r1.cells[idx].approxVal : r1.cells[idx].val;
+ Object o2 = useApprox ? r2.cells[idx].approxVal : r2.cells[idx].val;
+ int result = 0;
+ if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
+ else if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
+ else if (!o1.getClass().getName().equals(o2.getClass().getName())) {
+ throw new RuntimeException("Unable to compare " + o1 + " of type: " + o1.getClass().getName() + " from " + r1.cells[idx].toString() + " and " + o2 + " of type: " + o2.getClass().getName() + " from " + r2.cells[idx].toString());
+ }
+ return result == 0 ? (next == null ? 0 : next.compare(r1, r2, useApprox)) : sort.sortval * result;
+ }
+
+ //sets the new value according to precision in val_
+ void setApproxVal(List<Row> tmpMatrix) {
+ Object prevVal = null;
+ for (Row row : tmpMatrix) {
+ prevVal = row.cells[idx].approxVal =
+ (prevVal == null || Double.compare(Math.abs(((Number) prevVal).doubleValue() - ((Number) row.cells[idx].val).doubleValue()), precision) > 0) ?
+ row.cells[idx].val :
+ prevVal;
+ }
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (Object o : original.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ ew.put(String.valueOf(e.getKey()), e.getValue());
+ }
+ }
+
+ public Policy.SortParam getName() {
+ return name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
new file mode 100644
index 0000000..76c8c57
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.Utils;
+
+import static org.apache.solr.common.params.CoreAdminParams.NODE;
+
+
+public class Row implements MapWriter {
+ public final String node;
+ final Cell[] cells;
+ public Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas;
+ List<Clause> violations = new ArrayList<>();
+ boolean anyValueMissing = false;
+
+ public Row(String node, List<String> params, ClusterDataProvider dataProvider) {
+ collectionVsShardVsReplicas = dataProvider.getReplicaInfo(node, params);
+ if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>();
+ this.node = node;
+ cells = new Cell[params.size()];
+ Map<String, Object> vals = dataProvider.getNodeValues(node, params);
+ for (int i = 0; i < params.size(); i++) {
+ String s = params.get(i);
+ cells[i] = new Cell(i, s, Clause.validate(s,vals.get(s), false));
+ if (NODE.equals(s)) cells[i].val = node;
+ if (cells[i].val == null) anyValueMissing = true;
+ }
+ }
+
+ public Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, List<Clause> violations) {
+ this.node = node;
+ this.cells = new Cell[cells.length];
+ for (int i = 0; i < this.cells.length; i++) {
+ this.cells[i] = cells[i].copy();
+
+ }
+ this.anyValueMissing = anyValueMissing;
+ this.collectionVsShardVsReplicas = collectionVsShardVsReplicas;
+ this.violations = violations;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(node, (IteratorWriter) iw -> {
+ iw.add((MapWriter) e -> e.put("replicas", collectionVsShardVsReplicas));
+ for (Cell cell : cells) iw.add(cell);
+ });
+ }
+
+ Row copy() {
+ return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(collectionVsShardVsReplicas, 3), new ArrayList<>(violations));
+ }
+
+ Object getVal(String name) {
+ for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return node;
+ }
+
+ // this adds a replica to the replica info
+ public Row addReplica(String coll, String shard) {
+ Row row = copy();
+ Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
+ List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
+ replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, new HashMap<>()));
+ for (Cell cell : row.cells) {
+ if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() + 1;
+ }
+ return row;
+
+ }
+
+ public Pair<Row, ReplicaInfo> removeReplica(String coll, String shard) {
+ Row row = copy();
+ Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.get(coll);
+ if (c == null) return null;
+ List<ReplicaInfo> s = c.get(shard);
+ if (s == null || s.isEmpty()) return null;
+ for (Cell cell : row.cells) {
+ if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() -1;
+ }
+ return new Pair(row, s.remove(0));
+
+ }
+
+ public Cell[] getCells() {
+ return cells;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java
new file mode 100644
index 0000000..620f57d
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common classes for autoscaling parsing filtering nodes and sorting
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
index e40f32b..5b61822 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
@@ -32,10 +32,10 @@ import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
-import org.apache.solr.cloud.autoscaling.ClusterDataProvider;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 740de34..f039ba4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -46,7 +46,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
-import static org.apache.solr.cloud.autoscaling.Policy.POLICY;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM;
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
deleted file mode 100644
index 354851e..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.util.List;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.autoscaling.Policy.Suggester;
-
-class AddReplicaSuggester extends Suggester {
-
- SolrRequest init() {
- SolrRequest operation = tryEachNode(true);
- if (operation == null) operation = tryEachNode(false);
- return operation;
- }
-
- SolrRequest tryEachNode(boolean strict) {
- String coll = (String) hints.get(Hint.COLL);
- String shard = (String) hints.get(Hint.SHARD);
- if (coll == null || shard == null)
- throw new RuntimeException("add-replica requires 'collection' and 'shard'");
- //iterate through elements and identify the least loaded
-
- List<Clause.Violation> leastSeriousViolation = null;
- Integer targetNodeIndex = null;
- for (int i = getMatrix().size() - 1; i >= 0; i--) {
- Row row = getMatrix().get(i);
- if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
- Row tmpRow = row.addReplica(coll, shard);
- tmpRow.violations.clear();
-
- List<Clause.Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
- if(!containsNewErrors(errs)) {
- if(isLessSerious(errs, leastSeriousViolation)){
- leastSeriousViolation = errs;
- targetNodeIndex = i;
- }
- }
- }
-
- if (targetNodeIndex != null) {// there are no rule violations
- getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
- return CollectionAdminRequest
- .addReplicaToShard(coll, shard)
- .setNode(getMatrix().get(targetNodeIndex).node);
- }
-
- return null;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/13a3ae29/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
deleted file mode 100644
index 0f2b24b..0000000
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.util.Utils;
-
-class Cell implements MapWriter {
- final int index;
- final String name;
- Object val, approxVal;
-
- Cell(int index, String name, Object val) {
- this.index = index;
- this.name = name;
- this.val = val;
- }
-
- Cell(int index, String name, Object val, Object approxVal) {
- this.index = index;
- this.name = name;
- this.val = val;
- this.approxVal = approxVal;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(name, val);
- }
-
- @Override
- public String toString() {
- return Utils.toJSONString(this.toMap(new HashMap<>()));
- }
-
- public Cell copy() {
- return new Cell(index, name, val, approxVal);
- }
-}