You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/05/23 18:00:15 UTC
[1/8] lucene-solr:jira/solr-10515: SOLR-10373: Move setWT to
RequestHandlerUtils. Use JSON as default response writer for autoscaling API.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-10515 5aed0bfa9 -> 39464e2cb
SOLR-10373: Move setWT to RequestHandlerUtils. Use JSON as default response writer for autoscaling API.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab7257fe
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab7257fe
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab7257fe
Branch: refs/heads/jira/solr-10515
Commit: ab7257fedc66550b956632e3a23907b77691a7e2
Parents: 607184c
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue May 16 13:37:55 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue May 16 13:37:55 2017 +0530
----------------------------------------------------------------------
.../solr/cloud/autoscaling/AutoScalingHandler.java | 3 +++
.../src/java/org/apache/solr/handler/BlobHandler.java | 2 +-
.../org/apache/solr/handler/RequestHandlerUtils.java | 13 +++++++++++++
.../java/org/apache/solr/handler/SchemaHandler.java | 2 +-
.../org/apache/solr/handler/SolrConfigHandler.java | 11 +----------
.../apache/solr/handler/admin/SecurityConfHandler.java | 4 ++--
6 files changed, 21 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab7257fe/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 759ca14..701d499 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -52,6 +52,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
+import static org.apache.solr.common.params.CommonParams.JSON;
/**
* Handler for /cluster/autoscaling
@@ -83,6 +84,8 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
String httpMethod = (String) req.getContext().get("httpMethod");
+ RequestHandlerUtils.setWt(req, JSON);
+
if ("GET".equals(httpMethod)) {
Map<String, Object> map = zkReadAutoScalingConf(container.getZkController().getZkStateReader());
rsp.getValues().addAll(map);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab7257fe/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
index 052fcfe..ca0529a 100644
--- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -80,7 +80,7 @@ public class BlobHandler extends RequestHandlerBase implements PluginInfoInitial
public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
String httpMethod = req.getHttpMethod();
String path = (String) req.getContext().get("path");
- SolrConfigHandler.setWt(req, JSON);
+ RequestHandlerUtils.setWt(req, JSON);
List<String> pieces = StrUtils.splitSmart(path, '/');
String blobName = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab7257fe/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java
index 99f8315..4441024 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.*;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
@@ -119,4 +120,16 @@ public class RequestHandlerUtils
}
return false;
}
+
+ /**
+ * @since 6.7
+ */
+ public static void setWt(SolrQueryRequest req, String wt) {
+ SolrParams params = req.getParams();
+ if (params.get(CommonParams.WT) != null) return;//wt is set by user
+ Map<String, String> map = new HashMap<>(1);
+ map.put(CommonParams.WT, wt);
+ map.put("indent", "true");
+ req.setParams(SolrParams.wrapDefaults(params, new MapSolrParams(map)));
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab7257fe/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
index f3e503e..d37b388 100644
--- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
@@ -76,7 +76,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware,
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- SolrConfigHandler.setWt(req, JSON);
+ RequestHandlerUtils.setWt(req, JSON);
String httpMethod = (String) req.getContext().get("httpMethod");
if ("POST".equals(httpMethod)) {
if (isImmutableConfigSet) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab7257fe/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 08ce838..2ae2c14 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -119,7 +119,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- setWt(req, CommonParams.JSON);
+ RequestHandlerUtils.setWt(req, CommonParams.JSON);
String httpMethod = (String) req.getContext().get("httpMethod");
Command command = new Command(req, rsp, httpMethod);
if ("POST".equals(httpMethod)) {
@@ -668,15 +668,6 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
return null;
}
- public static void setWt(SolrQueryRequest req, String wt) {
- SolrParams params = req.getParams();
- if (params.get(CommonParams.WT) != null) return;//wt is set by user
- Map<String, String> map = new HashMap<>(1);
- map.put(CommonParams.WT, wt);
- map.put("indent", "true");
- req.setParams(SolrParams.wrapDefaults(params, new MapSolrParams(map)));
- }
-
@Override
public SolrRequestHandler getSubHandler(String path) {
if (subPaths.contains(path)) return this;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab7257fe/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java
index eceb4b7..aba1c04 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java
@@ -34,7 +34,7 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.RequestHandlerBase;
-import org.apache.solr.handler.SolrConfigHandler;
+import org.apache.solr.handler.RequestHandlerUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthenticationPlugin;
@@ -74,7 +74,7 @@ public abstract class SecurityConfHandler extends RequestHandlerBase implements
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- SolrConfigHandler.setWt(req, CommonParams.JSON);
+ RequestHandlerUtils.setWt(req, CommonParams.JSON);
String httpMethod = (String) req.getContext().get("httpMethod");
String path = (String) req.getContext().get("path");
String key = path.substring(path.lastIndexOf('/')+1);
[3/8] lucene-solr:jira/solr-10515: test all rows in one go instead of
one row at a time
Posted by ab...@apache.org.
test all rows in one go instead of one row at a time
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/07e630b7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/07e630b7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/07e630b7
Branch: refs/heads/jira/solr-10515
Commit: 07e630b7c4fc437816ea2c9cc612acf6492541a8
Parents: c15a173
Author: Noble Paul <no...@apache.org>
Authored: Mon May 22 15:40:49 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Mon May 22 15:40:49 2017 +0930
----------------------------------------------------------------------
.../cloud/autoscaling/AddReplicaSuggester.java | 16 ++-
.../org/apache/solr/cloud/autoscaling/Cell.java | 7 ++
.../apache/solr/cloud/autoscaling/Clause.java | 112 ++++++++++++++-----
.../cloud/autoscaling/MoveReplicaSuggester.java | 46 +++-----
.../apache/solr/cloud/autoscaling/Operand.java | 2 +-
.../apache/solr/cloud/autoscaling/Policy.java | 43 +++++--
.../solr/cloud/autoscaling/PolicyHelper.java | 15 ++-
.../org/apache/solr/cloud/autoscaling/Row.java | 13 ++-
.../solr/cloud/autoscaling/TestPolicy.java | 53 +++++----
9 files changed, 201 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
index fc0cecc..d0e510d 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/AddReplicaSuggester.java
@@ -17,6 +17,8 @@
package org.apache.solr.cloud.autoscaling;
+import java.util.List;
+
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.autoscaling.Policy.Suggester;
@@ -35,15 +37,16 @@ class AddReplicaSuggester extends Suggester {
if (coll == null || shard == null)
throw new RuntimeException("add-replica requires 'collection' and 'shard'");
//iterate through elements and identify the least loaded
+
for (int i = getMatrix().size() - 1; i >= 0; i--) {
Row row = getMatrix().get(i);
if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
- row = row.addReplica(coll, shard);
- row.violations.clear();
- for (Clause clause : session.expandedClauses) {
- if (strict || clause.strict) clause.test(row);
- }
- if (row.violations.isEmpty()) {// there are no rule violations
+ Row tmpRow = row.addReplica(coll, shard);
+ tmpRow.violations.clear();
+
+ List<Clause.Violation> errs = testChangedRow(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
+
+ if (!containsNewErrors(errs)) {// there are no rule violations
getMatrix().set(i, getMatrix().get(i).addReplica(coll, shard));
return CollectionAdminRequest
.addReplicaToShard(coll, shard)
@@ -53,4 +56,5 @@ class AddReplicaSuggester extends Suggester {
return null;
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
index c14490f..540872c 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
@@ -18,8 +18,10 @@
package org.apache.solr.cloud.autoscaling;
import java.io.IOException;
+import java.util.HashMap;
import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Utils;
class Cell implements MapWriter {
final int index;
@@ -44,6 +46,11 @@ class Cell implements MapWriter {
ew.put(name, val);
}
+ @Override
+ public String toString() {
+ return Utils.toJSONString(this.toMap(new HashMap<>()));
+ }
+
public Cell copy() {
return new Cell(index, name, val, val_);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
index 2da56e9..4fdd88a 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
@@ -18,16 +18,19 @@
package org.apache.solr.cloud.autoscaling;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Predicate;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.Utils;
import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
@@ -124,6 +127,10 @@ public class Clause implements MapWriter, Comparable<Clause> {
return op.match(val, row.getVal(name));
}
+ TestStatus match(Object testVal) {
+ return op.match(this.val, testVal);
+ }
+
boolean isPass(Object inputVal) {
return op.match(val, inputVal) == PASS;
}
@@ -169,44 +176,91 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
}
+ public class Violation {
+ final String shard, coll, node;
+ private final int hash;
- TestStatus test(Row row) {
- AtomicReference<TestStatus> result = new AtomicReference<>(NOT_APPLICABLE);
- if (isPerCollectiontag()) {
- for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.replicaInfo.entrySet()) {
- if (result.get() == FAIL) break;
- if (!collection.isPass(colls.getKey())) continue;
- int count = 0;
- for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
- if (!shard.isPass(shards.getKey()) || result.get() == FAIL) break;
- count += shards.getValue().size();
- if (shard.val.equals(EACH)) testReplicaCount(row, result, count);
- if (EACH.equals(shard.val)) count = 0;
+ private Violation(String coll, String shard, String node) {
+ this.shard = shard;
+ this.coll = coll;
+ this.node = node;
+ hash = ("" + coll + " " + shard + " " + node + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
+ }
+
+ public Clause getClause() {
+ return Clause.this;
+ }
+
+ @Override
+ public int hashCode() {
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that instanceof Violation) {
+ Violation ve = (Violation) that;
+ return Objects.equals(this.shard, (ve).shard) &&
+ Objects.equals(this.coll, (ve).coll) &&
+ Objects.equals(this.node, (ve).node);
+ }
+ return false;
+ }
+ }
+
+
+ public List<Violation> test(List<Row> allRows) {
+ List<Violation> errors = new ArrayList<>();
+ if (isPerCollectiontag()) {
+ Map<String, Map<String, Map<String, AtomicInteger>>> replicaCount = computeReplicaCounts(allRows);
+ for (Map.Entry<String, Map<String, Map<String, AtomicInteger>>> e : replicaCount.entrySet()) {
+ if (!collection.isPass(e.getKey())) continue;
+ for (Map.Entry<String, Map<String, AtomicInteger>> shardVsCount : e.getValue().entrySet()) {
+ if (!shard.isPass(shardVsCount.getKey())) continue;
+ for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
+ if (!replica.isPass(counts.getValue())) {
+ errors.add(new Violation(e.getKey(), shardVsCount.getKey(),
+ tag.name.equals("node") ? counts.getKey() : null));
+ }
+ }
}
- if (shard.val.equals(ANY)) testReplicaCount(row, result, count);
}
} else {
- if (!tag.isPass(row)) result.set(TestStatus.FAIL);
+ for (Row r : allRows) {
+ if (!tag.isPass(r)) {
+ errors.add(new Violation(null, null, r.node));
+ }
+ }
}
- if (result.get() == FAIL)
- row.violations.add(this);
- return result.get();
+ return errors;
}
- private void testReplicaCount(Row row, AtomicReference<TestStatus> result, int count) {
- if ("node".equals(tag.name)) if (!tag.isPass(row.node)) return;
- boolean checkCount = replica.op.match(replica.val, 0) != PASS || count > 0;
- if (replica.op == WILDCARD && count > 0 && !tag.isPass(row)) {//nodeRole:'!overseer', strict:false
- result.set(FAIL);
- } else if (checkCount && !replica.isPass(count)) {
- if (tag.op != WILDCARD && tag.isPass(row)) {
- result.set(FAIL);
- } else {
- result.set(FAIL);
+
+ private Map<String, Map<String, Map<String, AtomicInteger>>> computeReplicaCounts(List<Row> allRows) {
+ Map<String, Map<String, Map<String, AtomicInteger>>> replicaCount = new HashMap<>();
+ for (Row row : allRows)
+ for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.replicaInfo.entrySet()) {
+ String collectionName = colls.getKey();
+ if (!collection.isPass(collectionName)) continue;
+ replicaCount.putIfAbsent(collectionName, new HashMap<>());
+ Map<String, Map<String, AtomicInteger>> collMap = replicaCount.get(collectionName);
+ for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
+ String shardName = shards.getKey();
+ if (ANY.equals(shard.val)) shardName = ANY;
+ if (!shard.isPass(shardName)) break;
+ collMap.putIfAbsent(shardName, new HashMap<>());
+ Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
+ AtomicInteger count = null;
+ Object tagVal = row.getVal(tag.name);
+ if (tag.isPass(tagVal)) {
+ tagVsCount.put(String.valueOf(tagVal), count = tagVsCount.getOrDefault(tagVal, new AtomicInteger()));
+ count.addAndGet(shards.getValue().size());
+ }
+ }
}
- }
+ return replicaCount;
}
public boolean isStrict() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
index bcc039c..2c295e2 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
@@ -17,19 +17,13 @@
package org.apache.solr.cloud.autoscaling;
-import java.util.Map;
+import java.util.List;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.autoscaling.Clause.Violation;
import org.apache.solr.cloud.autoscaling.Policy.Suggester;
import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.Utils;
-
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
-import static org.apache.solr.common.params.CoreAdminParams.NODE;
-import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
public class MoveReplicaSuggester extends Suggester {
@@ -53,29 +47,21 @@ public class MoveReplicaSuggester extends Suggester {
continue;
}
tmpRow.violations.clear();
- for (Clause clause : session.expandedClauses) {
- if (strict || clause.strict) {
- clause.test(tmpRow);
- }
- }
+
final int i = getMatrix().indexOf(fromRow);
- if (tmpRow.violations.isEmpty()) {
- for (int j = getMatrix().size() - 1; j > i; j--) {
- Row targetRow = getMatrix().get(j);
- if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
- targetRow = targetRow.addReplica(coll, shard);
- targetRow.violations.clear();
- for (Clause clause : session.expandedClauses) {
- if (strict || clause.strict) clause.test(targetRow);
- }
- if (targetRow.violations.isEmpty()) {
- getMatrix().set(i, getMatrix().get(i).removeReplica(coll, shard).first());
- getMatrix().set(j, getMatrix().get(j).addReplica(coll, shard));
- return new CollectionAdminRequest.MoveReplica(
- coll,
- pair.second().name,
- targetRow.node);
- }
+ for (int j = getMatrix().size() - 1; j > i; j--) {
+ Row targetRow = getMatrix().get(j);
+ if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
+ targetRow = targetRow.addReplica(coll, shard);
+ targetRow.violations.clear();
+ List<Violation> errs = testChangedRow(strict, getModifiedMatrix(getModifiedMatrix(getMatrix(), tmpRow, i), targetRow, j));
+ if (!containsNewErrors(errs)) {
+ getMatrix().set(i, getMatrix().get(i).removeReplica(coll, shard).first());
+ getMatrix().set(j, getMatrix().get(j).addReplica(coll, shard));
+ return new CollectionAdminRequest.MoveReplica(
+ coll,
+ pair.second().name,
+ targetRow.node);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
index 13c2589..6ff1ef5 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Operand.java
@@ -85,7 +85,7 @@ public enum Operand {
return operand + expectedVal.toString();
}
- Object checkNumeric(Object val) {
+ Integer checkNumeric(Object val) {
if (val == null) return null;
try {
return Integer.parseInt(val.toString());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index 69cb0f1..11a6250 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -34,6 +34,7 @@ import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.cloud.autoscaling.Clause.Violation;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
@@ -122,6 +123,7 @@ public class Policy implements MapWriter {
final List<Row> matrix;
Set<String> collections = new HashSet<>();
List<Clause> expandedClauses;
+ List<Violation> violations = new ArrayList<>();
private List<String> paramsOfInterest;
private Session(List<String> nodes, ClusterDataProvider dataProvider,
@@ -199,16 +201,13 @@ public class Policy implements MapWriter {
}
for (Clause clause : expandedClauses) {
- for (Row row : matrix) {
- clause.test(row);
- }
- }
+ List<Violation> errs = clause.test(matrix);
+ violations.addAll(errs);
+ }
}
- public Map<String, List<Clause>> getViolations() {
- return matrix.stream()
- .filter(row -> !row.violations.isEmpty())
- .collect(Collectors.toMap(r -> r.node, r -> r.violations));
+ public List<Violation> getViolations() {
+ return violations;
}
public Suggester getSuggester(CollectionAction action) {
@@ -302,6 +301,7 @@ public class Policy implements MapWriter {
protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
Policy.Session session;
SolrRequest operation;
+ protected List<Violation> originalViolations = new ArrayList<>();
private boolean isInitialized = false;
private void _init(Session session) {
@@ -318,6 +318,8 @@ public class Policy implements MapWriter {
public SolrRequest getOperation() {
if (!isInitialized) {
+ session.applyRules();
+ originalViolations.addAll(session.getViolations());
this.operation = init();
isInitialized = true;
}
@@ -332,6 +334,13 @@ public class Policy implements MapWriter {
return session.matrix;
}
+ boolean containsNewErrors(List<Clause.Violation> errs){
+ for (Clause.Violation err : errs) {
+ if(!originalViolations.contains(err)) return true;
+ }
+ return false;
+ }
+
List<Pair<ReplicaInfo, Row>> getValidReplicas(boolean sortDesc, boolean isSource, int until) {
List<Pair<Policy.ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
@@ -356,6 +365,24 @@ public class Policy implements MapWriter {
}
}
}
+ protected List<Violation> testChangedRow(boolean strict,List<Row> rows) {
+ List<Violation> errors = new ArrayList<>();
+ for (Clause clause : session.expandedClauses) {
+ if (strict || clause.strict) {
+ List<Violation> errs = clause.test(rows);
+ if (!errs.isEmpty()) {
+ errors.addAll(errs);
+ }
+ }
+ }
+ return errors;
+ }
+
+ ArrayList<Row> getModifiedMatrix(List<Row> matrix ,Row tmpRow, int i) {
+ ArrayList<Row> copy = new ArrayList<>(matrix);
+ copy.set(i, tmpRow);
+ return copy;
+ }
protected boolean isAllowed(Object v, Hint hint) {
Object hintVal = hints.get(hint);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
index e7ad25e..2012be8 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
@@ -23,6 +23,9 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.common.SolrException;
@@ -51,7 +54,13 @@ public class PolicyHelper {
@Override
public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return delegate.getReplicaInfo(node, keys);
+ Map<String, Map<String, List<Policy.ReplicaInfo>>> replicaInfo = delegate.getReplicaInfo(node, keys);
+ for (String s : optionalPolicyMapping.keySet()) {
+ if (!replicaInfo.containsKey(s)) {
+ replicaInfo.put(s, new HashMap<>());
+ }
+ }
+ return replicaInfo;
}
@Override
@@ -79,10 +88,10 @@ public class PolicyHelper {
.hint(Hint.SHARD, shardName);
SolrRequest op = suggester.getOperation();
if (op == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(policy));
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(Utils.getDeepCopy(session.expandedClauses, 4, true)));
}
session = suggester.getSession();
- positionMapping.get(shardName).add((String) op.getParams().get(CoreAdminParams.NODE));
+ positionMapping.get(shardName).add(op.getParams().get(CoreAdminParams.NODE));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
index d5f94c9..b9304ba 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
@@ -93,9 +93,12 @@ class Row implements MapWriter {
Row row = copy();
Map<String, List<ReplicaInfo>> c = row.replicaInfo.get(coll);
if (c == null) row.replicaInfo.put(coll, c = new HashMap<>());
- List<ReplicaInfo> s = c.get(shard);
- if (s == null) c.put(shard, s = new ArrayList<>());
- s.add(new ReplicaInfo(""+new Random().nextInt(1000)+1000,coll,shard, new HashMap<>()));
+ List<ReplicaInfo> replicas = c.get(shard);
+ if (replicas == null) c.put(shard, replicas = new ArrayList<>());
+ replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, new HashMap<>()));
+ for (Cell cell : row.cells) {
+ if (cell.name.equals("cores")) cell.val = ((Number) cell.val).intValue() + 1;
+ }
return row;
}
@@ -103,10 +106,10 @@ class Row implements MapWriter {
Pair<Row, ReplicaInfo> removeReplica(String coll, String shard) {
Row row = copy();
Map<String, List<ReplicaInfo>> c = row.replicaInfo.get(coll);
- if(c == null) return null;
+ if (c == null) return null;
List<ReplicaInfo> s = c.get(shard);
if (s == null || s.isEmpty()) return null;
- return new Pair(row,s.remove(0));
+ return new Pair(row, s.remove(0));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/07e630b7/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index def2d47..9d1f12a 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -28,7 +28,9 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.ImmutableList;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.MoveReplica;
import org.apache.solr.cloud.autoscaling.Policy.Suggester.Hint;
@@ -76,7 +78,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
" { 'minimize': 'cores', 'precision': 50}" +
" ]," +
" 'cluster-policy': [" +
- " { 'replica': '#ANY', 'nodeRole': '!overseer'}," +
+ " { 'replica': 0, 'nodeRole': 'overseer'}," +
" { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}" +
" ]," +
" 'policies': {" +
@@ -92,17 +94,17 @@ public class TestPolicy extends SolrTestCaseJ4 {
Collections.sort(clauses);
assertEquals(clauses.size(), 4);
assertEquals("1", String.valueOf(clauses.get(0).original.get("replica")));
- assertEquals("<2", String.valueOf(clauses.get(1).original.get("replica")));
+ assertEquals("0", String.valueOf(clauses.get(1).original.get("replica")));
assertEquals("#ANY", clauses.get(3).original.get("shard"));
- assertEquals("rack1",clauses.get(1).original.get("rack"));
- assertEquals("!overseer", clauses.get(2).original.get("nodeRole"));
+ assertEquals("rack1",clauses.get(2).original.get("rack"));
+ assertEquals("overseer", clauses.get(1).original.get("nodeRole"));
}
public void testConditionsSort(){
String rules = "{" +
" 'cluster-policy':[" +
- " { 'nodeRole':'!overseer', 'strict':false}," +
+ " { 'nodeRole':'overseer', replica: 0, 'strict':false}," +
" { 'replica':'<1', 'node':'node3', 'shard':'#EACH'}," +
" { 'replica':'<2', 'node':'#ANY', 'shard':'#EACH'}," +
" { 'replica':1, 'rack':'rack1'}]" +
@@ -110,7 +112,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
Policy p = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
List<Clause> clauses = new ArrayList<>(p.getClusterPolicy());
Collections.sort(clauses);
- assertEquals("rack", clauses.get(0).tag.name);
+ assertEquals("nodeRole", clauses.get(0).tag.name);
+ assertEquals("rack", clauses.get(1).tag.name);
}
public static String clusterState = "{'gettingstarted':{" +
" 'router':{'name':'compositeId'}," +
@@ -157,7 +160,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
public void testRules() throws IOException {
String rules = "{" +
"cluster-policy:[" +
- "{nodeRole:'!overseer', strict:false}," +
+ "{nodeRole:'overseer',replica : 0 , strict:false}," +
"{replica:'<1',node:node3}," +
"{replica:'<2',node:'#ANY', shard:'#EACH'}]," +
" cluster-preferences:[" +
@@ -183,7 +186,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("node2", l.get(3).node);
- Map<String, List<Clause>> violations = session.getViolations();
+ /*Map<String, List<Clause>> violations = session.getViolations();
System.out.println(Utils.getDeepCopy(violations, 6));
assertEquals(3, violations.size());
List<Clause> v = violations.get("node4");
@@ -197,7 +200,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(v);
assertEquals(v.get(0).replica.op, Operand.LESS_THAN);
assertEquals(v.get(0).replica.val, 1);
- assertEquals(v.get(0).tag.val, "node3");
+ assertEquals(v.get(0).tag.val, "node3");*/
Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
.hint(Hint.COLL, "gettingstarted")
.hint(Hint.SHARD, "r1");
@@ -212,11 +215,11 @@ public class TestPolicy extends SolrTestCaseJ4 {
"node4:{cores:8, freedisk: 375, heap:16900, nodeRole:overseer}" +
"}");
session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
- operation = session.getSuggester(MOVEREPLICA)
+ SolrRequest opReq = session.getSuggester(MOVEREPLICA)
.hint(Hint.TARGET_NODE, "node5")
- .getOperation()
- .getParams();
- assertEquals("node5", operation.get("targetNode"));
+ .getOperation();
+ assertNotNull(opReq);
+ assertEquals("node5", opReq.getParams().get("targetNode"));
}
@@ -226,7 +229,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
" 'cluster-policy':[" +
" {'cores':'<10','node':'#ANY'}," +
" {'replica':'<3','shard':'#EACH','node':'#ANY'}," +
- " {'nodeRole':'!overseer','replica':'#ANY'}]," +
+ " {'nodeRole':'overseer','replica':'0'}]," +
" 'cluster-preferences':[" +
" {'minimize':'cores', 'precision':3}," +
" {'maximize':'freedisk','precision':100}]}";
@@ -372,21 +375,22 @@ public class TestPolicy extends SolrTestCaseJ4 {
};
}
-/* public void testMultiReplicaPlacement() {
+ public void testMultiReplicaPlacement() {
String autoScaleJson ="{" +
" 'cluster-preferences': [" +
- " { 'minimize': 'freedisk', 'precision': 50}" +
+ " { maximize : freedisk , precision: 50}," +
+ " { minimize : cores, precision: 2}" +
" ]," +
" 'cluster-policy': [" +
- " { 'nodeRole': '!overseer'}," +
- " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'" +
+ " { replica : '0' , 'nodeRole': 'overseer'}," +
+ " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'" +
" }" +
" ]," +
" 'policies': {" +
" 'policy1': [" +
- " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'}," +
- " { 'replica': '<2', 'shard': '#EACH', 'rack': 'rack1'}," +
- " { 'replica': '1', 'sysprop.fs': 'ssd', 'shard': '#EACH'}" +
+ " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " { 'replica': '<2', 'shard': '#EACH', 'rack': 'rack1'}" +
+// " { 'replica': '1', 'sysprop.fs': 'ssd', 'shard': '#EACH'}" +
" ]" +
" }" +
"}";
@@ -396,7 +400,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"node1:{cores:12, freedisk: 334, heap:10480, rack:rack3}," +
"node2:{cores:4, freedisk: 749, heap:6873, sysprop.fs : ssd, rack:rack1}," +
"node3:{cores:7, freedisk: 262, heap:7834, rack:rack4}," +
- "node4:{cores:8, freedisk: 375, heap:16900, nodeRole:overseer, rack:rack2}" +
+ "node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, rack:rack2}" +
"}");
ClusterDataProvider dataProvider = new ClusterDataProvider() {
@@ -425,10 +429,11 @@ public class TestPolicy extends SolrTestCaseJ4 {
Map<String, List<String>> locations = PolicyHelper.getReplicaLocations(
"newColl", (Map<String, Object>) Utils.fromJSONString(autoScaleJson),
dataProvider, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3);
- System.out.println(Utils.toJSONString(locations));
+ assertTrue(locations.get("shard1").containsAll(ImmutableList.of("node2","node1","node3")));
+ assertTrue(locations.get("shard2").containsAll(ImmutableList.of("node2","node1","node3")));
- }*/
+ }
public static Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaDetails(String node, String s) {
ValidatingJsonMap m = ValidatingJsonMap
[7/8] lucene-solr:jira/solr-10515: Merge branch 'feature/autoscaling'
into jira/solr-10515
Posted by ab...@apache.org.
Merge branch 'feature/autoscaling' into jira/solr-10515
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c252c555
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c252c555
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c252c555
Branch: refs/heads/jira/solr-10515
Commit: c252c555f3004fcc318640162289f3e995d7209e
Parents: 5aed0bf ba670a0
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue May 23 12:03:17 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue May 23 12:03:17 2017 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../cloud/autoscaling/AutoScalingHandler.java | 141 ++++++++++++-------
.../autoscaling/AutoScalingHandlerTest.java | 89 +++++++++++-
.../solrj/impl/SolrClientDataProvider.java | 21 ++-
.../cloud/autoscaling/AddReplicaSuggester.java | 16 ++-
.../org/apache/solr/cloud/autoscaling/Cell.java | 7 +
.../apache/solr/cloud/autoscaling/Clause.java | 124 ++++++++++++----
.../cloud/autoscaling/MoveReplicaSuggester.java | 46 +++---
.../apache/solr/cloud/autoscaling/Operand.java | 2 +-
.../apache/solr/cloud/autoscaling/Policy.java | 49 +++++--
.../solr/cloud/autoscaling/PolicyHelper.java | 15 +-
.../org/apache/solr/cloud/autoscaling/Row.java | 13 +-
.../java/org/apache/solr/common/MapWriter.java | 6 +
.../solr/common/cloud/rule/ImplicitSnitch.java | 4 +-
.../solr/cloud/autoscaling/TestPolicy.java | 100 +++++++------
15 files changed, 442 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
[5/8] lucene-solr:jira/solr-10515: test for violations
Posted by ab...@apache.org.
test for violations
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/11f70a0e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/11f70a0e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/11f70a0e
Branch: refs/heads/jira/solr-10515
Commit: 11f70a0e20769db075c67ef7a17a502e3b2fac3f
Parents: e59cce6
Author: Noble Paul <no...@apache.org>
Authored: Tue May 23 15:04:02 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Tue May 23 15:04:02 2017 +0930
----------------------------------------------------------------------
.../apache/solr/cloud/autoscaling/Clause.java | 20 +++++++++++++++----
.../java/org/apache/solr/common/MapWriter.java | 6 ++++++
.../solr/cloud/autoscaling/TestPolicy.java | 21 ++++++--------------
3 files changed, 28 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/11f70a0e/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
index 4fdd88a..d466ce9 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
@@ -176,15 +176,17 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
}
- public class Violation {
+ public class Violation implements MapWriter {
final String shard, coll, node;
+ final Object actualVal;
private final int hash;
- private Violation(String coll, String shard, String node) {
+ private Violation(String coll, String shard, String node, Object actualVal) {
this.shard = shard;
this.coll = coll;
this.node = node;
+ this.actualVal = actualVal;
hash = ("" + coll + " " + shard + " " + node + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
}
@@ -207,6 +209,16 @@ public class Clause implements MapWriter, Comparable<Clause> {
}
return false;
}
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.putIfNotNull("collection", coll);
+ ew.putIfNotNull("shard", shard);
+ ew.putIfNotNull("node", node);
+ ew.putIfNotNull("violation", (MapWriter) ew1 -> ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
+ String.valueOf(actualVal)));
+ ew.put("clause", getClause());
+ }
}
@@ -221,7 +233,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
if (!replica.isPass(counts.getValue())) {
errors.add(new Violation(e.getKey(), shardVsCount.getKey(),
- tag.name.equals("node") ? counts.getKey() : null));
+ tag.name.equals("node") ? counts.getKey() : null, counts.getValue()));
}
}
}
@@ -229,7 +241,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
} else {
for (Row r : allRows) {
if (!tag.isPass(r)) {
- errors.add(new Violation(null, null, r.node));
+ errors.add(new Violation(null, null, r.node, r.getVal(tag.name) ));
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/11f70a0e/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index 24307d6..fca6e2b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -64,6 +64,12 @@ public interface MapWriter extends MapSerializable {
*/
EntryWriter put(String k, Object v) throws IOException;
+ default EntryWriter putIfNotNull(String k, Object v) throws IOException {
+ if(v != null) put(k,v);
+ return this;
+ }
+
+
default EntryWriter put(String k, int v) throws IOException {
put(k, (Integer) v);
return this;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/11f70a0e/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 9d1f12a..5f49bea 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest.MoveReplica;
+import org.apache.solr.cloud.autoscaling.Clause.Violation;
import org.apache.solr.cloud.autoscaling.Policy.Suggester.Hint;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.SolrParams;
@@ -186,21 +186,12 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("node2", l.get(3).node);
- /*Map<String, List<Clause>> violations = session.getViolations();
- System.out.println(Utils.getDeepCopy(violations, 6));
+ List<Violation> violations = session.getViolations();
assertEquals(3, violations.size());
- List<Clause> v = violations.get("node4");
- assertNotNull(v);
- assertEquals(v.get(0).tag.name, "nodeRole");
- v = violations.get("node1");
- assertNotNull(v);
- assertEquals(v.get(0).replica.op, Operand.LESS_THAN);
- assertEquals(v.get(0).replica.val, 2);
- v = violations.get("node3");
- assertNotNull(v);
- assertEquals(v.get(0).replica.op, Operand.LESS_THAN);
- assertEquals(v.get(0).replica.val, 1);
- assertEquals(v.get(0).tag.val, "node3");*/
+ assertTrue( violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.val)));
+ assertTrue( violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.name)));
+ assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.op == Operand.LESS_THAN && "node".equals(violation.getClause().tag.name))));
+
Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
.hint(Hint.COLL, "gettingstarted")
.hint(Hint.SHARD, "r1");
[8/8] lucene-solr:jira/solr-10515: SOLR-10515 Fix enqueue/dequeue,
add javadoc and some debug logging.
Posted by ab...@apache.org.
SOLR-10515 Fix enqueue/dequeue, add javadoc and some debug logging.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/39464e2c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/39464e2c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/39464e2c
Branch: refs/heads/jira/solr-10515
Commit: 39464e2cb8dab8ad99e7e01d7b4589e948f6ff8b
Parents: c252c55
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue May 23 19:59:34 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue May 23 19:59:34 2017 +0200
----------------------------------------------------------------------
.../solr/cloud/autoscaling/AutoScaling.java | 3 +-
.../cloud/autoscaling/NodeAddedTrigger.java | 16 ++++----
.../solr/cloud/autoscaling/NodeLostTrigger.java | 12 +++---
.../cloud/autoscaling/ScheduledTriggers.java | 39 +++++++++++++-------
.../solr/cloud/autoscaling/TriggerBase.java | 10 +++--
.../autoscaling/TriggerIntegrationTest.java | 23 +++++++-----
6 files changed, 62 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39464e2c/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index 0d6da79..4beffd8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@ -86,7 +86,8 @@ public class AutoScaling {
* This method is executed when a trigger is ready to fire.
*
* @param event a subclass of {@link TriggerEvent}
- * @return true if the listener was ready to perform actions on the event, false otherwise.
+ * @return true if the listener was ready to perform actions on the event, false
+ * otherwise. If false was returned then callers should assume the event was discarded.
*/
boolean triggerFired(E event);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39464e2c/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index d783f86..c3bbffe 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@ -29,6 +29,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -58,7 +60,7 @@ public class NodeAddedTrigger extends TriggerBase<NodeAddedTrigger.NodeAddedEven
private Set<String> lastLiveNodes;
- private Map<String, Long> nodeNameVsTimeAdded = new HashMap<>();
+ private Map<String, Long> nodeNameVsTimeAdded = new TreeMap<>();
public NodeAddedTrigger(String name, Map<String, Object> properties,
CoreContainer container) {
@@ -78,7 +80,7 @@ public class NodeAddedTrigger extends TriggerBase<NodeAddedTrigger.NodeAddedEven
} else {
actions = Collections.emptyList();
}
- lastLiveNodes = container.getZkController().getZkStateReader().getClusterState().getLiveNodes();
+ lastLiveNodes = new TreeSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
this.enabled = (boolean) properties.getOrDefault("enabled", true);
this.waitForSecond = ((Long) properties.getOrDefault("waitFor", -1L)).intValue();
@@ -155,8 +157,8 @@ public class NodeAddedTrigger extends TriggerBase<NodeAddedTrigger.NodeAddedEven
if (old instanceof NodeAddedTrigger) {
NodeAddedTrigger that = (NodeAddedTrigger) old;
assert this.name.equals(that.name);
- this.lastLiveNodes = new HashSet<>(that.lastLiveNodes);
- this.nodeNameVsTimeAdded = new HashMap<>(that.nodeNameVsTimeAdded);
+ this.lastLiveNodes = new TreeSet<>(that.lastLiveNodes);
+ this.nodeNameVsTimeAdded = new TreeMap<>(that.nodeNameVsTimeAdded);
} else {
throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
"Unable to restore state from an unknown type of trigger");
@@ -199,10 +201,6 @@ public class NodeAddedTrigger extends TriggerBase<NodeAddedTrigger.NodeAddedEven
ZkStateReader reader = container.getZkController().getZkStateReader();
Set<String> newLiveNodes = reader.getClusterState().getLiveNodes();
log.debug("Found livenodes: {}", newLiveNodes);
- if (lastLiveNodes == null) {
- lastLiveNodes = newLiveNodes;
- return;
- }
// have any nodes that we were tracking been removed from the cluster?
// if so, remove them from the tracking map
@@ -238,7 +236,7 @@ public class NodeAddedTrigger extends TriggerBase<NodeAddedTrigger.NodeAddedEven
}
}
- lastLiveNodes = newLiveNodes;
+ lastLiveNodes = new TreeSet(newLiveNodes);
} catch (RuntimeException e) {
log.error("Unexpected exception in NodeAddedTrigger", e);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39464e2c/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 2024a30..feea94d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -29,6 +29,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
@@ -58,7 +60,7 @@ public class NodeLostTrigger extends TriggerBase<NodeLostTrigger.NodeLostEvent>
private Set<String> lastLiveNodes;
- private Map<String, Long> nodeNameVsTimeRemoved = new HashMap<>();
+ private Map<String, Long> nodeNameVsTimeRemoved = new TreeMap<>();
public NodeLostTrigger(String name, Map<String, Object> properties,
CoreContainer container) {
@@ -78,7 +80,7 @@ public class NodeLostTrigger extends TriggerBase<NodeLostTrigger.NodeLostEvent>
} else {
actions = Collections.emptyList();
}
- lastLiveNodes = container.getZkController().getZkStateReader().getClusterState().getLiveNodes();
+ lastLiveNodes = new TreeSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
this.enabled = (boolean) properties.getOrDefault("enabled", true);
this.waitForSecond = ((Long) properties.getOrDefault("waitFor", -1L)).intValue();
@@ -154,8 +156,8 @@ public class NodeLostTrigger extends TriggerBase<NodeLostTrigger.NodeLostEvent>
if (old instanceof NodeLostTrigger) {
NodeLostTrigger that = (NodeLostTrigger) old;
assert this.name.equals(that.name);
- this.lastLiveNodes = new HashSet<>(that.lastLiveNodes);
- this.nodeNameVsTimeRemoved = new HashMap<>(that.nodeNameVsTimeRemoved);
+ this.lastLiveNodes = new TreeSet<>(that.lastLiveNodes);
+ this.nodeNameVsTimeRemoved = new TreeMap<>(that.nodeNameVsTimeRemoved);
} else {
throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
"Unable to restore state from an unknown type of trigger");
@@ -230,7 +232,7 @@ public class NodeLostTrigger extends TriggerBase<NodeLostTrigger.NodeLostEvent>
}
}
- lastLiveNodes = newLiveNodes;
+ lastLiveNodes = new TreeSet<>(newLiveNodes);
} catch (RuntimeException e) {
log.error("Unexpected exception in NodeLostTrigger", e);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39464e2c/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
index f14bcb7..3e94d1c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
@@ -123,23 +123,22 @@ public class ScheduledTriggers implements Closeable {
ScheduledTrigger scheduledSource = scheduledTriggers.get(event.getSource());
if (scheduledSource == null) {
log.warn("Ignoring autoscaling event " + event + " because the source trigger: " + event.getSource() + " doesn't exist.");
- // XXX not sure what to return here...
- return true;
+ return false;
}
boolean replaying = event.getProperty(TriggerEventBase.REPLAYING) != null ? (Boolean)event.getProperty(TriggerEventBase.REPLAYING) : false;
- final boolean enqueued;
- if (replaying) {
- enqueued = false;
- } else {
- enqueued = scheduledTrigger.enqueue(event);
- }
AutoScaling.Trigger source = scheduledSource.trigger;
if (source.isClosed()) {
log.warn("Ignoring autoscaling event " + event + " because the source trigger: " + source + " has already been closed");
- // we do not want to lose this event just because the trigger were closed, perhaps a replacement will need it
+ // we do not want to lose this event just because the trigger was closed, perhaps a replacement will need it
return false;
}
if (hasPendingActions.compareAndSet(false, true)) {
+ final boolean enqueued;
+ if (replaying) {
+ enqueued = false;
+ } else {
+ enqueued = scheduledTrigger.enqueue(event);
+ }
List<TriggerAction> actions = source.getActions();
if (actions != null) {
actionExecutor.submit(() -> {
@@ -165,6 +164,12 @@ public class ScheduledTriggers implements Closeable {
hasPendingActions.set(false);
}
});
+ } else {
+ if (enqueued) {
+ AutoScaling.TriggerEvent ev = scheduledTrigger.dequeue();
+ assert ev.getId().equals(event.getId());
+ }
+ hasPendingActions.set(false);
}
return true;
} else {
@@ -244,10 +249,10 @@ public class ScheduledTriggers implements Closeable {
if (isClosed) {
throw new AlreadyClosedException("ScheduledTrigger " + trigger.getName() + " has been closed.");
}
- log.debug("-- run scheduled trigger " + trigger.getName());
+ log.debug("--scheduled trigger " + trigger.getName());
// replay accumulated events on first run, if any
if (replay) {
- log.debug("--replaying...");
+ log.debug(" --replaying...");
AutoScaling.TriggerEvent event;
// peek first without removing - we may crash before calling the listener
while ((event = queue.peekEvent()) != null) {
@@ -256,17 +261,23 @@ public class ScheduledTriggers implements Closeable {
if (! trigger.getListener().triggerFired(event)) {
log.error("Failed to re-play event, discarding: " + event);
}
- log.debug("--replayed event: " + event);
+ log.debug(" --replayed event: " + event);
queue.pollEvent(); // always remove it from queue
}
// now restore saved state to possibly generate new events from old state on the first run
- trigger.restoreState();
+ try {
+ trigger.restoreState();
+ } catch (Exception e) {
+ // log but don't throw - see below
+ log.error("Error restoring trigger state " + trigger.getName(), e);
+ }
replay = false;
}
// fire a trigger only if an action is not pending
// note this is not fool proof e.g. it does not prevent an action being executed while a trigger
// is still executing. There is additional protection against that scenario in the event listener.
if (!hasPendingActions.get()) {
+ log.debug(" --run " + trigger.getName());
try {
trigger.run();
} catch (Exception e) {
@@ -277,6 +288,8 @@ public class ScheduledTriggers implements Closeable {
// checkpoint after each run
trigger.saveState();
}
+ } else {
+ log.debug(" --hasPendingActions - skipping " + trigger.getName());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39464e2c/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
index b94824c..235cc9d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
@@ -17,7 +17,9 @@
package org.apache.solr.cloud.autoscaling;
import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -63,9 +65,10 @@ public abstract class TriggerBase<E extends AutoScaling.TriggerEvent> implements
@Override
public void saveState() {
Map<String,Object> state = getState();
- byte[] data = Utils.toJSON(state);
+ TreeMap<String, Object> map = new TreeMap<>(state);
+ byte[] data = Utils.toJSON(map);
// skip saving if identical
- if (lastState != null && lastState.equals(data)) {
+ if (lastState != null && Arrays.equals(lastState, data)) {
LOG.debug("--skip saving " + getName());
return;
}
@@ -89,7 +92,6 @@ public abstract class TriggerBase<E extends AutoScaling.TriggerEvent> implements
public void restoreState() {
byte[] data = null;
String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + getName();
- LOG.info("--restore state: " + path);
try {
if (zkClient.exists(path, true)) {
data = zkClient.getData(path, null, new Stat(), true);
@@ -99,7 +101,7 @@ public abstract class TriggerBase<E extends AutoScaling.TriggerEvent> implements
}
if (data != null) {
Map<String, Object> state = (Map<String, Object>) Utils.fromJSON(data);
- LOG.info(" -- state: " + state);
+ LOG.info("-- restored state of " + path + ": " + state);
setState(state);
lastState = data;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39464e2c/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index 235d748..ac22500 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -189,7 +189,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
}
}
- if (!triggerFiredLatch.await(10, TimeUnit.SECONDS)) {
+ if (!triggerFiredLatch.await(20, TimeUnit.SECONDS)) {
fail("Both triggers should have fired by now");
}
}
@@ -465,7 +465,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
// stop the overseer, somebody else will take over as the overseer
cluster.stopJettySolrRunner(index);
-
+ Thread.sleep(10000);
JettySolrRunner newNode = cluster.startJettySolrRunner();
boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
assertTrue("The trigger did not fire at all", await);
@@ -495,14 +495,19 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Override
public void process(AutoScaling.TriggerEvent event) {
- if (triggerFired.compareAndSet(false, true)) {
- eventRef.set(event);
- if (System.nanoTime() - event.getEventTime() <= TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS)) {
- fail("NodeAddedListener was fired before the configured waitFor period");
+ try {
+ if (triggerFired.compareAndSet(false, true)) {
+ eventRef.set(event);
+ if (System.nanoTime() - event.getEventTime() <= TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS)) {
+ fail("NodeAddedListener was fired before the configured waitFor period");
+ }
+ triggerFiredLatch.countDown();
+ } else {
+ fail("NodeAddedTrigger was fired more than once!");
}
- triggerFiredLatch.countDown();
- } else {
- fail("NodeAddedTrigger was fired more than once!");
+ } catch (Throwable t) {
+ log.info("--throwable", t);
+ throw t;
}
}
[2/8] lucene-solr:jira/solr-10515: SOLR-10714: OverseerTriggerThread
does not start triggers on overseer start until autoscaling config watcher is
fired
Posted by ab...@apache.org.
SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling config watcher is fired
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c15a1737
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c15a1737
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c15a1737
Branch: refs/heads/jira/solr-10515
Commit: c15a17379ff7daea0ab1cb8e89e67745e82e68fc
Parents: ab7257f
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri May 19 17:51:18 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri May 19 17:51:18 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../solr/cloud/autoscaling/AutoScaling.java | 2 +
.../autoscaling/OverseerTriggerThread.java | 160 +++++++++++--------
.../autoscaling/TriggerIntegrationTest.java | 56 ++++++-
4 files changed, 149 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c15a1737/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 211ed14..03f2f39 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -101,6 +101,9 @@ Bug Fixes
* SOLR-10602: Triggers should be able to restore state from old instances when taking over. (shalin)
+* SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling
+ config watcher is fired. (shalin)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c15a1737/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index cd08ea9..5c992b0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Preconditions;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.solr.core.CoreContainer;
@@ -123,6 +124,7 @@ public class AutoScaling {
private boolean isClosed = false;
public TriggerFactory(CoreContainer coreContainer) {
+ Preconditions.checkNotNull(coreContainer);
this.coreContainer = coreContainer;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c15a1737/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
index eca8c0b..cd619c1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
@@ -54,6 +54,8 @@ public class OverseerTriggerThread implements Runnable, Closeable {
private final ZkStateReader zkStateReader;
+ private final SolrZkClient zkClient;
+
private final ScheduledTriggers scheduledTriggers;
private final AutoScaling.TriggerFactory triggerFactory;
@@ -74,6 +76,7 @@ public class OverseerTriggerThread implements Runnable, Closeable {
public OverseerTriggerThread(ZkController zkController) {
this.zkController = zkController;
zkStateReader = zkController.getZkStateReader();
+ zkClient = zkController.getZkClient();
scheduledTriggers = new ScheduledTriggers();
triggerFactory = new AutoScaling.TriggerFactory(zkController.getCoreContainer());
}
@@ -90,13 +93,27 @@ public class OverseerTriggerThread implements Runnable, Closeable {
}
IOUtils.closeQuietly(triggerFactory);
IOUtils.closeQuietly(scheduledTriggers);
+ log.debug("OverseerTriggerThread has been closed explicitly");
}
@Override
public void run() {
int lastZnodeVersion = znodeVersion;
- SolrZkClient zkClient = zkStateReader.getZkClient();
- createWatcher(zkClient);
+
+ try {
+ refreshAutoScalingConf(new AutoScalingWatcher());
+ } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
+ log.warn("ZooKeeper watch triggered for autoscaling conf, but Solr cannot talk to ZK: [{}]", e.getMessage());
+ } catch (KeeperException e) {
+ log.error("A ZK error has occurred", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("Interrupted", e);
+ } catch (Exception e) {
+ log.error("Unexpected exception", e);
+ }
while (true) {
Map<String, AutoScaling.Trigger> copy = null;
@@ -104,18 +121,31 @@ public class OverseerTriggerThread implements Runnable, Closeable {
// this can throw InterruptedException and we don't want to unlock if it did, so we keep this outside
// of the try/finally block
updateLock.lockInterruptibly();
+
+ // must check for close here before we await on the condition otherwise we can only be woken up on interruption
+ if (isClosed) {
+ log.warn("OverseerTriggerThread has been closed, exiting.");
+ break;
+ }
+
+ log.debug("Current znodeVersion {}, lastZnodeVersion {}", znodeVersion, lastZnodeVersion);
+
try {
if (znodeVersion == lastZnodeVersion) {
updated.await();
// are we closed?
- if (isClosed) break;
+ if (isClosed) {
+ log.warn("OverseerTriggerThread woken up but we are closed, exiting.");
+ break;
+ }
// spurious wakeup?
if (znodeVersion == lastZnodeVersion) continue;
- lastZnodeVersion = znodeVersion;
}
copy = new HashMap<>(activeTriggers);
+ lastZnodeVersion = znodeVersion;
+ log.debug("Processed trigger updates upto znodeVersion {}", znodeVersion);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
@@ -145,77 +175,65 @@ public class OverseerTriggerThread implements Runnable, Closeable {
}
}
- private void createWatcher(SolrZkClient zkClient) {
+ class AutoScalingWatcher implements Watcher {
+ @Override
+ public void process(WatchedEvent watchedEvent) {
+ // session events are not change events, and do not remove the watcher
+ if (Event.EventType.None.equals(watchedEvent.getType())) {
+ return;
+ }
+
+ try {
+ refreshAutoScalingConf(this);
+ } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
+ log.warn("ZooKeeper watch triggered for autoscaling conf, but Solr cannot talk to ZK: [{}]", e.getMessage());
+ } catch (KeeperException e) {
+ log.error("A ZK error has occurred", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("Interrupted", e);
+ } catch (Exception e) {
+ log.error("Unexpected exception", e);
+ }
+ }
+
+ }
+
+ private void refreshAutoScalingConf(Watcher watcher) throws KeeperException, InterruptedException {
+ updateLock.lock();
try {
- zkClient.exists(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, new Watcher() {
- @Override
- public void process(WatchedEvent watchedEvent) {
- // session events are not change events, and do not remove the watcher
- if (Event.EventType.None.equals(watchedEvent.getType())) {
- return;
- }
+ if (isClosed) {
+ return;
+ }
+ final Stat stat = new Stat();
+ final byte[] data = zkClient.getData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, watcher, stat, true);
+ log.debug("Refreshing {} with znode version {}", ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, stat.getVersion());
+ if (znodeVersion >= stat.getVersion()) {
+ // protect against reordered watcher fires by ensuring that we only move forward
+ return;
+ }
+ znodeVersion = stat.getVersion();
+ Map<String, AutoScaling.Trigger> triggerMap = loadTriggers(triggerFactory, data);
- try {
- refreshAndWatch();
- } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
- log.warn("ZooKeeper watch triggered for autoscaling conf, but Solr cannot talk to ZK: [{}]", e.getMessage());
- } catch (KeeperException e) {
- log.error("A ZK error has occurred", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- log.warn("Interrupted", e);
- } catch (Exception e) {
- log.error("Unexpected exception", e);
- }
- }
+ // remove all active triggers that have been removed from ZK
+ Set<String> trackingKeySet = activeTriggers.keySet();
+ trackingKeySet.retainAll(triggerMap.keySet());
- private void refreshAndWatch() throws KeeperException, InterruptedException {
- updateLock.lock();
- try {
- if (isClosed) {
- return;
- }
- final Stat stat = new Stat();
- final byte[] data = zkClient.getData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, this, stat, true);
- log.debug("{} watcher fired with znode version {}", ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, stat.getVersion());
- if (znodeVersion >= stat.getVersion()) {
- // protect against reordered watcher fires by ensuring that we only move forward
- return;
- }
- znodeVersion = stat.getVersion();
- Map<String, AutoScaling.Trigger> triggerMap = loadTriggers(triggerFactory, data);
-
- // remove all active triggers that have been removed from ZK
- Set<String> trackingKeySet = activeTriggers.keySet();
- trackingKeySet.retainAll(triggerMap.keySet());
-
- // now lets add or remove triggers which have been enabled or disabled respectively
- for (Map.Entry<String, AutoScaling.Trigger> entry : triggerMap.entrySet()) {
- String triggerName = entry.getKey();
- AutoScaling.Trigger trigger = entry.getValue();
- if (trigger.isEnabled()) {
- activeTriggers.put(triggerName, trigger);
- } else {
- activeTriggers.remove(triggerName);
- }
- }
- updated.signalAll();
- } finally {
- updateLock.unlock();
- }
+ // now lets add or remove triggers which have been enabled or disabled respectively
+ for (Map.Entry<String, AutoScaling.Trigger> entry : triggerMap.entrySet()) {
+ String triggerName = entry.getKey();
+ AutoScaling.Trigger trigger = entry.getValue();
+ if (trigger.isEnabled()) {
+ activeTriggers.put(triggerName, trigger);
+ } else {
+ activeTriggers.remove(triggerName);
}
- }, true);
- } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
- log.error("OverseerTriggerThread could not talk to ZooKeeper", e);
- } catch (KeeperException e) {
- log.error("Exception in OverseerTriggerThread", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- log.error("OverseerTriggerThread interrupted", e);
+ }
+ updated.signalAll();
+ } finally {
+ updateLock.unlock();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c15a1737/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index 545c0d6..0ac8183 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -33,6 +33,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.util.NamedList;
@@ -73,7 +74,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
}
@Before
- public void setupTest() throws KeeperException, InterruptedException, IOException, SolrServerException {
+ public void setupTest() throws Exception {
waitForSeconds = 1 + random().nextInt(3);
actionCreated = new CountDownLatch(1);
triggerFiredLatch = new CountDownLatch(1);
@@ -85,6 +86,11 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
// todo nocommit -- add testing for the v2 path
// String path = random().nextBoolean() ? "/admin/autoscaling" : "/v2/cluster/autoscaling";
this.path = "/admin/autoscaling";
+ while (cluster.getJettySolrRunners().size() < 2) {
+ // perhaps a test stopped a node but didn't start it back
+ // lets start a node
+ cluster.startJettySolrRunner();
+ }
}
@Test
@@ -416,6 +422,54 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
lostNodeName, nodeLostEvent.getNodeName());
}
+ @Test
+ public void testContinueTriggersOnOverseerRestart() throws Exception {
+ CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus();
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ CollectionAdminResponse adminResponse = status.process(solrClient);
+ NamedList<Object> response = adminResponse.getResponse();
+ String leader = (String) response.get("leader");
+ JettySolrRunner overseerNode = null;
+ int index = -1;
+ List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
+ for (int i = 0; i < jettySolrRunners.size(); i++) {
+ JettySolrRunner runner = jettySolrRunners.get(i);
+ if (runner.getNodeName().equals(leader)) {
+ overseerNode = runner;
+ index = i;
+ break;
+ }
+ }
+ assertNotNull(overseerNode);
+
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = new AutoScalingHandlerTest.AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ if (!actionCreated.await(3, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ // stop the overseer, somebody else will take over as the overseer
+ cluster.stopJettySolrRunner(index);
+
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) eventRef.get();
+ assertNotNull(nodeAddedEvent);
+ assertEquals("The node added trigger was fired but for a different node",
+ newNode.getNodeName(), nodeAddedEvent.getNodeName());
+ }
+
public static class TestTriggerAction implements TriggerAction {
public TestTriggerAction() {
[6/8] lucene-solr:jira/solr-10515: SOLR-10677: Expose a diagnostics
API to return nodes sorted by load in descending order and any policy
violations
Posted by ab...@apache.org.
SOLR-10677: Expose a diagnostics API to return nodes sorted by load in descending order and any policy violations
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ba670a0c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ba670a0c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ba670a0c
Branch: refs/heads/jira/solr-10515
Commit: ba670a0c2abcbeaae0b630bb46d73a1b6db04e11
Parents: 11f70a0
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue May 23 14:58:04 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue May 23 14:58:04 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../cloud/autoscaling/AutoScalingHandler.java | 141 ++++++++++++-------
.../autoscaling/AutoScalingHandlerTest.java | 89 +++++++++++-
.../solrj/impl/SolrClientDataProvider.java | 21 ++-
.../apache/solr/cloud/autoscaling/Policy.java | 6 +-
.../solr/common/cloud/rule/ImplicitSnitch.java | 2 +
.../solr/cloud/autoscaling/TestPolicy.java | 30 ++--
7 files changed, 218 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba670a0c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 03f2f39..ecfeedc 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -91,6 +91,9 @@ New Features
* SOLR-10373: Implement read API for autoscaling configuration at /admin/autoscaling or
/cluster/autoscaling paths. (shalin)
+* SOLR-10677: Expose a diagnostics API to return nodes sorted by load in descending order and
+ any policy violations. (shalin)
+
Bug Fixes
----------------------
* SOLR-9262: Connection and read timeouts are being ignored by UpdateShardHandler after SOLR-4509.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba670a0c/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 701d499..7617f13 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud.autoscaling;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
@@ -33,6 +34,8 @@ import java.util.concurrent.TimeUnit;
import com.google.common.collect.ImmutableSet;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -83,58 +86,98 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- String httpMethod = (String) req.getContext().get("httpMethod");
- RequestHandlerUtils.setWt(req, JSON);
-
- if ("GET".equals(httpMethod)) {
- Map<String, Object> map = zkReadAutoScalingConf(container.getZkController().getZkStateReader());
- rsp.getValues().addAll(map);
- RequestHandlerUtils.addExperimentalFormatWarning(rsp);
- } else {
- if (req.getContentStreams() == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No commands specified for autoscaling");
- }
- List<CommandOperation> ops = CommandOperation.readCommands(req.getContentStreams(), rsp, singletonCommands);
- if (ops == null) {
- // errors have already been added to the response so there's nothing left to do
- return;
+ try {
+ String httpMethod = (String) req.getContext().get("httpMethod");
+ RequestHandlerUtils.setWt(req, JSON);
+
+ if ("GET".equals(httpMethod)) {
+ Map<String, Object> map = zkReadAutoScalingConf(container.getZkController().getZkStateReader());
+ rsp.getValues().addAll(map);
+ if (req.getParams().getBool("diagnostics", false)) {
+ handleDiagnostics(rsp, map);
+ }
+ } else {
+ if (req.getContentStreams() == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No commands specified for autoscaling");
+ }
+ List<CommandOperation> ops = CommandOperation.readCommands(req.getContentStreams(), rsp, singletonCommands);
+ if (ops == null) {
+ // errors have already been added to the response so there's nothing left to do
+ return;
+ }
+ for (CommandOperation op : ops) {
+ switch (op.name) {
+ case "set-trigger":
+ handleSetTrigger(req, rsp, op);
+ break;
+ case "remove-trigger":
+ handleRemoveTrigger(req, rsp, op);
+ break;
+ case "set-listener":
+ handleSetListener(req, rsp, op);
+ break;
+ case "remove-listener":
+ handleRemoveListener(req, rsp, op);
+ break;
+ case "suspend-trigger":
+ handleSuspendTrigger(req, rsp, op);
+ break;
+ case "resume-trigger":
+ handleResumeTrigger(req, rsp, op);
+ break;
+ case "set-policy":
+ handleSetPolicies(req, rsp, op);
+ break;
+ case "remove-policy":
+ handleRemovePolicy(req, rsp, op);
+ break;
+ case "set-cluster-preferences":
+ handleSetClusterPreferences(req, rsp, op);
+ break;
+ case "set-cluster-policy":
+ handleSetClusterPolicy(req, rsp, op);
+ break;
+ default:
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown command: " + op.name);
+ }
+ }
}
- for (CommandOperation op : ops) {
- switch (op.name) {
- case "set-trigger":
- handleSetTrigger(req, rsp, op);
- break;
- case "remove-trigger":
- handleRemoveTrigger(req, rsp, op);
- break;
- case "set-listener":
- handleSetListener(req, rsp, op);
- break;
- case "remove-listener":
- handleRemoveListener(req, rsp, op);
- break;
- case "suspend-trigger":
- handleSuspendTrigger(req, rsp, op);
- break;
- case "resume-trigger":
- handleResumeTrigger(req, rsp, op);
- break;
- case "set-policy":
- handleSetPolicies(req, rsp, op);
- break;
- case "remove-policy":
- handleRemovePolicy(req, rsp, op);
- break;
- case "set-cluster-preferences":
- handleSetClusterPreferences(req, rsp, op);
- break;
- case "set-cluster-policy":
- handleSetClusterPolicy(req, rsp, op);
- break;
- default:
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown command: " + op.name);
+ } finally {
+ RequestHandlerUtils.addExperimentalFormatWarning(rsp);
+ }
+ }
+
+ private void handleDiagnostics(SolrQueryResponse rsp, Map<String, Object> autoScalingConf) throws IOException {
+ Policy policy = new Policy(autoScalingConf);
+ try (CloudSolrClient build = new CloudSolrClient.Builder()
+ .withHttpClient(container.getUpdateShardHandler().getHttpClient())
+ .withZkHost(container.getZkController().getZkServerAddress()).build()) {
+ Policy.Session session = policy.createSession(new SolrClientDataProvider(build));
+ List<Row> sorted = session.getSorted();
+ List<Clause.Violation> violations = session.getViolations();
+
+ List<Preference> clusterPreferences = policy.getClusterPreferences();
+
+ List<Map<String, Object>> sortedNodes = new ArrayList<>(sorted.size());
+ for (Row row : sorted) {
+ Map<String, Object> map = Utils.makeMap("node", row.node);
+ for (Cell cell : row.cells) {
+ for (Preference clusterPreference : clusterPreferences) {
+ Policy.SortParam name = clusterPreference.name;
+ if (cell.name.equalsIgnoreCase(name.name())) {
+ map.put(name.name(), cell.val);
+ break;
+ }
+ }
}
+ sortedNodes.add(map);
}
+
+ Map<String, Object> map = new HashMap<>(2);
+ map.put("sortedNodes", sortedNodes);
+
+ map.put("violations", violations);
+ rsp.getValues().add("diagnostics", map);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba670a0c/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 81ef5ff..460fe64 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -29,6 +29,8 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -396,7 +398,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
" 'set-cluster-preferences': [" +
" {'minimize': 'cores', 'precision': 3}," +
" {'maximize': 'freedisk','precision': 100}," +
- " {'minimize': 'cpu','precision': 10}]" +
+ " {'minimize': 'sysLoadAvg','precision': 10}]" +
"}";
req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPreferencesCommand);
response = solrClient.request(req);
@@ -409,7 +411,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
// set preferences
setPreferencesCommand = "{" +
" 'set-cluster-preferences': [" +
- " {'minimize': 'cpu','precision': 10}]" +
+ " {'minimize': 'sysLoadAvg','precision': 10}]" +
"}";
req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPreferencesCommand);
response = solrClient.request(req);
@@ -466,7 +468,8 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
" 'set-cluster-preferences': [" +
" {'minimize': 'cores', 'precision': 3}," +
" {'maximize': 'freedisk','precision': 100}," +
- " {'minimize': 'cpu','precision': 10}]" +
+ " {'minimize': 'sysLoadAvg','precision': 10}," +
+ " {'minimize': 'heapUsage','precision': 10}]" +
"}";
req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setPreferencesCommand);
response = solrClient.request(req);
@@ -478,7 +481,6 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
" {'nodeRole':'!overseer', 'replica':'#ANY'}" +
" ]," +
" 'policy1':[" +
- " {'cores':'<2', 'node':'#ANY'}," +
" {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
" ]" +
"}}";
@@ -486,7 +488,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
- SolrQuery query = new SolrQuery().setParam(CommonParams.QT, path);
+ SolrQuery query = new SolrQuery().setParam(CommonParams.QT, path).setParam("diagnostics", true);
QueryResponse queryResponse = solrClient.query(query);
response = queryResponse.getResponse();
@@ -502,7 +504,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
List<Map> clusterPrefs = (List<Map>) response.get("cluster-preferences");
assertNotNull(clusterPrefs);
- assertEquals(3, clusterPrefs.size());
+ assertEquals(4, clusterPrefs.size());
List<Map> clusterPolicy = (List<Map>) response.get("cluster-policy");
assertNotNull(clusterPolicy);
@@ -513,6 +515,81 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertEquals(2, policies.size());
assertNotNull(policies.get("xyz"));
assertNotNull(policies.get("policy1"));
+
+ Map<String, Object> diagnostics = (Map<String, Object>) response.get("diagnostics");
+ List sortedNodes = (List) diagnostics.get("sortedNodes");
+ assertNotNull(sortedNodes);
+
+ assertEquals(2, sortedNodes.size());
+ String[] sortedNodeNames = new String[2];
+ for (int i = 0; i < 2; i++) {
+ Map node = (Map) sortedNodes.get(i);
+ assertNotNull(node);
+ assertEquals(5, node.size());
+ assertNotNull(sortedNodeNames[i] = (String) node.get("node"));
+ assertNotNull(node.get("cores"));
+ assertEquals(0, node.get("cores"));
+ assertNotNull(node.get("freedisk"));
+ assertNotNull(node.get("sysLoadAvg"));
+ assertNotNull(node.get("heapUsage"));
+ }
+
+ List<Map<String, Object>> violations = (List<Map<String, Object>>) diagnostics.get("violations");
+ assertNotNull(violations);
+ assertEquals(0, violations.size());
+
+ // lets create a collection and ensure that its details show up in the diagnostics output
+ CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTest", 1, 2);
+ CollectionAdminResponse adminResponse = create.process(solrClient);
+ assertTrue(adminResponse.isSuccess());
+
+ // get the diagnostics output again
+ queryResponse = solrClient.query(query);
+ response = queryResponse.getResponse();
+ diagnostics = (Map<String, Object>) response.get("diagnostics");
+ sortedNodes = (List) diagnostics.get("sortedNodes");
+ assertNotNull(sortedNodes);
+
+ assertEquals(2, sortedNodes.size());
+ for (int i = 0; i < 2; i++) {
+ Map node = (Map) sortedNodes.get(i);
+ assertNotNull(node);
+ assertEquals(5, node.size());
+ assertNotNull(node.get("node"));
+ assertEquals(sortedNodeNames[i], node.get("node"));
+ assertNotNull(node.get("cores"));
+ assertEquals(1, node.get("cores"));
+ assertNotNull(node.get("freedisk"));
+ assertNotNull(node.get("sysLoadAvg"));
+ assertNotNull(node.get("heapUsage"));
+ }
+
+ violations = (List<Map<String, Object>>) diagnostics.get("violations");
+ assertNotNull(violations);
+ assertEquals(0, violations.size());
+
+ // lets create a collection which violates the rule replicas < 2
+ create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", 1, 6);
+ create.setMaxShardsPerNode(10);
+ adminResponse = create.process(solrClient);
+ assertTrue(adminResponse.isSuccess());
+
+ // get the diagnostics output again
+ queryResponse = solrClient.query(query);
+ response = queryResponse.getResponse();
+ diagnostics = (Map<String, Object>) response.get("diagnostics");
+ sortedNodes = (List) diagnostics.get("sortedNodes");
+ assertNotNull(sortedNodes);
+
+ violations = (List<Map<String, Object>>) diagnostics.get("violations");
+ assertNotNull(violations);
+ assertEquals(2, violations.size());
+ for (Map<String, Object> violation : violations) {
+ assertEquals("readApiTestViolations", violation.get("collection"));
+ assertEquals("shard1", violation.get("shard"));
+ assertEquals(Utils.makeMap("replica", "3"), violation.get("violation"));
+ assertNotNull(violation.get("clause"));
+ }
}
static class AutoScalingRequest extends SolrRequest {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba670a0c/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
index 03a652d..0e5e3d1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
@@ -167,7 +167,7 @@ public class SolrClientDataProvider implements ClusterDataProvider, MapWriter {
@Override
protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
ClientSnitchCtx snitchContext = (ClientSnitchCtx) ctx;
- List<String> groups = new ArrayList<>();
+ Set<String> groups = new HashSet<>();
List<String> prefixes = new ArrayList<>();
if (requestedTags.contains(DISK)) {
groups.add("solr.node");
@@ -177,6 +177,14 @@ public class SolrClientDataProvider implements ClusterDataProvider, MapWriter {
groups.add("solr.core");
prefixes.add("CORE.coreName");
}
+ if (requestedTags.contains(SYSLOADAVG)) {
+ groups.add("solr.jvm");
+ prefixes.add("os.systemLoadAverage");
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ groups.add("solr.jvm");
+ prefixes.add("memory.heap.usage");
+ }
if(groups.isEmpty() || prefixes.isEmpty()) return;
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -188,7 +196,7 @@ public class SolrClientDataProvider implements ClusterDataProvider, MapWriter {
Map m = rsp.nl.asMap(4);
if(requestedTags.contains(DISK)){
Number n = (Number) Utils.getObjectByPath(m,true, "metrics/solr.node/CONTAINER.fs.usableSpace/value");
- if(n != null) ctx.getTags().put(DISK, n.longValue());
+ if(n != null) ctx.getTags().put(DISK, n.doubleValue() / 1024.0d / 1024.0d / 1024.0d);
}
if(requestedTags.contains(CORES)){
int count = 0;
@@ -198,7 +206,14 @@ public class SolrClientDataProvider implements ClusterDataProvider, MapWriter {
}
ctx.getTags().put(CORES, count);
}
-
+ if (requestedTags.contains(SYSLOADAVG)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.jvm/os.systemLoadAverage/value");
+ if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue() * 100.0d);
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.jvm/memory.heap.usage/value");
+ if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
+ }
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", e);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba670a0c/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index 11a6250..de82fd5 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -100,6 +100,10 @@ public class Policy implements MapWriter {
return clusterPolicy;
}
+ public List<Preference> getClusterPreferences() {
+ return clusterPreferences;
+ }
+
@Override
public void writeMap(EntryWriter ew) throws IOException {
if (!policies.isEmpty()) {
@@ -241,7 +245,7 @@ public class Policy implements MapWriter {
}
enum SortParam {
- freedisk, cores, heap, cpu;
+ freedisk, cores, heapUsage, sysLoadAvg;
static SortParam get(String m) {
for (SortParam p : values()) if (p.name().equals(m)) return p;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba670a0c/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
index d55df39..8e596dc 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
@@ -48,6 +48,8 @@ public class ImplicitSnitch extends Snitch {
public static final String ROLE = "role";
public static final String NODEROLE = "nodeRole";
public static final String SYSPROP = "sysprop.";
+ public static final String SYSLOADAVG = "sysLoadAvg";
+ public static final String HEAPUSAGE = "heapUsage";
public static final List<String> IP_SNITCHES = Collections.unmodifiableList(Arrays.asList("ip_1", "ip_2", "ip_3", "ip_4"));
public static final Set<String> tags = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(NODE, PORT, HOST, CORES, DISK, ROLE, "ip_1", "ip_2", "ip_3", "ip_4")));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba670a0c/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 5f49bea..36c18e0 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -166,13 +166,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
" cluster-preferences:[" +
"{minimize:cores , precision:2}," +
"{maximize:freedisk, precision:50}, " +
- "{minimize:heap, precision:1000}]}";
+ "{minimize:heapUsage, precision:1000}]}";
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heap:10480}," +
- "node2:{cores:4, freedisk: 749, heap:6873}," +
- "node3:{cores:7, freedisk: 262, heap:7834}," +
- "node4:{cores:8, freedisk: 375, heap:16900, nodeRole:overseer}" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
"}");
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
@@ -199,11 +199,11 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("node2", operation.get("node"));
nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heap:10480}," +
- "node2:{cores:4, freedisk: 749, heap:6873}," +
- "node3:{cores:7, freedisk: 262, heap:7834}," +
- "node5:{cores:0, freedisk: 895, heap:17834}," +
- "node4:{cores:8, freedisk: 375, heap:16900, nodeRole:overseer}" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834}," +
+ "node5:{cores:0, freedisk: 895, heapUsage:17834}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" +
"}");
session = policy.createSession(getClusterDataProvider(nodeValues, clusterState));
SolrRequest opReq = session.getSuggester(MOVEREPLICA)
@@ -282,7 +282,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"'cluster-preferences':[" +
"{'minimize':'cores','precision':2}," +
"{'maximize':'freedisk','precision':50}," +
- "{'minimize':'heap','precision':1000}" +
+ "{'minimize':'heapUsage','precision':1000}" +
"]," +
"'cluster-policy':[" +
"{'nodeRole':'!overseer','strict':false}," +
@@ -300,10 +300,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}";
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
- "node1:{cores:12, freedisk: 334, heap:10480, rack: rack4}," +
- "node2:{cores:4, freedisk: 749, heap:6873, rack: rack3}," +
- "node3:{cores:7, freedisk: 262, heap:7834, rack: rack2}," +
- "node4:{cores:8, freedisk: 375, heap:16900, nodeRole:overseer, rack: rack1}" +
+ "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," +
+ "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," +
+ "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2}," +
+ "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" +
"}");
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(rules));
ClusterDataProvider clusterDataProvider = getClusterDataProvider(nodeValues, clusterState);
[4/8] lucene-solr:jira/solr-10515: nodeROle misspelled
Posted by ab...@apache.org.
nodeROle misspelled
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e59cce6a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e59cce6a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e59cce6a
Branch: refs/heads/jira/solr-10515
Commit: e59cce6a873b9740c99c0bf8a1d89ca50b839372
Parents: 07e630b
Author: Noble Paul <no...@apache.org>
Authored: Tue May 23 12:14:02 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Tue May 23 12:14:02 2017 +0930
----------------------------------------------------------------------
.../src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e59cce6a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
index 3cae572..d55df39 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
@@ -46,7 +46,7 @@ public class ImplicitSnitch extends Snitch {
public static final String CORES = "cores";
public static final String DISK = "freedisk";
public static final String ROLE = "role";
- public static final String NODEROLE = "noderole";
+ public static final String NODEROLE = "nodeRole";
public static final String SYSPROP = "sysprop.";
public static final List<String> IP_SNITCHES = Collections.unmodifiableList(Arrays.asList("ip_1", "ip_2", "ip_3", "ip_4"));
public static final Set<String> tags = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(NODE, PORT, HOST, CORES, DISK, ROLE, "ip_1", "ip_2", "ip_3", "ip_4")));