You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/05/27 21:59:03 UTC
[lucene-solr] branch branch_8x updated: SOLR-13493:
/autoscaling/suggestions to be able to filter by type (#686)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new c445ad4 SOLR-13493: /autoscaling/suggestions to be able to filter by type (#686)
c445ad4 is described below
commit c445ad4c75f14d3ce71c4d0bb6eeea78f210aae5
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Tue May 28 07:20:12 2019 +1000
SOLR-13493: /autoscaling/suggestions to be able to filter by type (#686)
* SOLR-13493: /autoscaling/suggestions to be able to filter by type
---
solr/CHANGES.txt | 2 +
.../solr/cloud/autoscaling/AutoScalingHandler.java | 15 ++---
.../test/org/apache/solr/cloud/CloudTestUtils.java | 20 +++++--
.../cloud/autoscaling/AutoScalingHandlerTest.java | 8 +++
.../src/solrcloud-autoscaling-api.adoc | 8 +++
.../solrj/cloud/autoscaling/PolicyHelper.java | 66 ++++++++++++++--------
.../client/solrj/cloud/autoscaling/Suggester.java | 6 +-
.../client/solrj/cloud/autoscaling/Suggestion.java | 9 ++-
.../solrj/cloud/autoscaling/TestPolicy2.java | 6 +-
9 files changed, 97 insertions(+), 43 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b764402..186ba9b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -52,6 +52,8 @@ New Features
* SOLR-13484: autoscaling/diagnostics API should be able to give diagnostics output from config sent as a payload (noble)
+* SOLR-13493: /autoscaling/suggestions to be able to filter by type (noble)
+
Bug Fixes
----------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 135b0e4..b6755b1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -51,6 +51,7 @@ import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.StrUtils;
@@ -104,12 +105,12 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
DEFAULT_ACTIONS.add(map);
}
- Optional<BiConsumer<SolrQueryResponse, AutoScalingConfig>> getSubpathExecutor(List<String> path) {
+ Optional<BiConsumer<SolrQueryResponse, AutoScalingConfig>> getSubpathExecutor(List<String> path, SolrQueryRequest request) {
if (path.size() == 3) {
if (DIAGNOSTICS.equals(path.get(2))) {
- return Optional.of(this::handleDiagnostics);
+ return Optional.of((rsp, autoScalingConf) -> handleDiagnostics(rsp, autoScalingConf));
} else if (SUGGESTIONS.equals(path.get(2))) {
- return Optional.of(this::handleSuggestions);
+ return Optional.of((rsp, autoScalingConf) -> handleSuggestions(rsp, autoScalingConf, request.getParams()));
} else {
return Optional.empty();
}
@@ -145,7 +146,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
});
} else {
- getSubpathExecutor(parts).ifPresent(it -> it.accept(rsp, autoScalingConf));
+ getSubpathExecutor(parts, req).ifPresent(it -> it.accept(rsp, autoScalingConf));
}
} else {
if (req.getContentStreams() == null) {
@@ -155,7 +156,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
if (path != null) {
List<String> parts = StrUtils.splitSmart(path, '/', true);
if(parts.size() == 3){
- getSubpathExecutor(parts).ifPresent(it -> {
+ getSubpathExecutor(parts, req).ifPresent(it -> {
Map map = null;
try {
map = (Map) Utils.fromJSON(req.getContentStreams().iterator().next().getStream());
@@ -186,9 +187,9 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
- private void handleSuggestions(SolrQueryResponse rsp, AutoScalingConfig autoScalingConf) {
+ private void handleSuggestions(SolrQueryResponse rsp, AutoScalingConfig autoScalingConf, SolrParams params) {
rsp.getValues().add("suggestions",
- PolicyHelper.getSuggestions(autoScalingConf, cloudManager));
+ PolicyHelper.getSuggestions(autoScalingConf, cloudManager, params));
}
public void processOps(SolrQueryRequest req, SolrQueryResponse rsp, List<CommandOperation> ops)
diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java b/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
index 196cb20..f86a6ba 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
@@ -118,7 +118,7 @@ public class CloudTestUtils {
* Helper class for sending (JSON) autoscaling requests that can randomize between V1 and V2 requests
*/
public static class AutoScalingRequest extends SolrRequest {
-
+ private SolrParams params = null;
/**
* Creates a request using a randomized root path (V1 vs V2)
*
@@ -137,6 +137,10 @@ public class CloudTestUtils {
* @param message JSON payload, may be null
*/
public static SolrRequest create(SolrRequest.METHOD m, String subPath, String message) {
+ return create(m,subPath,message,null);
+
+ }
+ public static SolrRequest create(SolrRequest.METHOD m, String subPath, String message, SolrParams params) {
final boolean useV1 = LuceneTestCase.random().nextBoolean();
String path = useV1 ? "/admin/autoscaling" : "/cluster/autoscaling";
if (null != subPath) {
@@ -144,10 +148,11 @@ public class CloudTestUtils {
path += subPath;
}
return useV1
- ? new AutoScalingRequest(m, path, message)
- : new V2Request.Builder(path).withMethod(m).withPayload(message).build();
+ ? new AutoScalingRequest(m, path, message).withParams(params)
+ : new V2Request.Builder(path).withMethod(m).withParams(params).withPayload(message).build();
+
}
-
+
protected final String message;
/**
@@ -161,9 +166,14 @@ public class CloudTestUtils {
this.message = message;
}
+
+ AutoScalingRequest withParams(SolrParams params){
+ this.params = params;
+ return this;
+ }
@Override
public SolrParams getParams() {
- return null;
+ return params;
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index d950c3c..5b98e44 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -20,6 +20,7 @@ package org.apache.solr.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
@@ -40,6 +41,7 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.AutoScalingParams;
+import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
@@ -51,6 +53,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.repair;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
import static org.apache.solr.common.util.Utils.getObjectByPath;
@@ -124,6 +127,11 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
}
});
assertTrue(passed[0]);
+
+ req = AutoScalingRequest.create(SolrRequest.METHOD.POST, "/suggestions", configPayload, new MapSolrParams(Collections.singletonMap("type", repair.name())));
+ response = solrClient.request(req);
+ assertTrue(((Collection) response.get("suggestions")).isEmpty());
+
CollectionAdminRequest.deleteCollection(COLLNAME)
.process(cluster.getSolrClient());
}
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc
index acbfe16..a30b3ae 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc
@@ -318,6 +318,14 @@ The API is available at `/admin/autoscaling/suggestions`. Here is an example out
The suggested `operation` is an API call that can be invoked to remedy the current violation.
+The types of suggestions available are
+
+* `violation` : Fixes a violation to one or more policy rules
+* `repair` : Add missing replicas
+* `improvement` : move replicas around so that the load is more evenly balanced according to the autoscaling preferences
+
+By default, the suggestions API return all of the above , in that order. However it is possible to fetch only certain types by adding a request parameter `type`. e.g: `type=violation&type=repair`
+
=== Inline Policy Configuration
If there is no autoscaling policy configured or if you wish to use a configuration other than the default, it is possible to send the autoscaling policy JSON as an inline payload as follows:
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 5a9acb4..955295a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
@@ -46,6 +47,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
@@ -55,6 +57,10 @@ import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.improvement;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.repair;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.unresolved_violation;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.violation;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
import static org.apache.solr.common.ConditionalMapWriter.dedupeKeyPredicate;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
@@ -229,46 +235,60 @@ public class PolicyHelper {
}
});
}
+ public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
+ SolrCloudManager cloudManager, SolrParams params) {
+ return getSuggestions(autoScalingConf, cloudManager, 20, 10, params);
+ }
public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
SolrCloudManager cloudManager) {
- return getSuggestions(autoScalingConf, cloudManager, 20, 10);
+ return getSuggestions(autoScalingConf, cloudManager, 20, 10, null);
}
+
public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
- SolrCloudManager cloudManager, int max, int timeoutInSecs) {
+ SolrCloudManager cloudManager, int max, int timeoutInSecs, SolrParams params) {
Policy policy = autoScalingConf.getPolicy();
Suggestion.Ctx ctx = new Suggestion.Ctx();
ctx.endTime = cloudManager.getTimeSource().getTimeNs() + TimeUnit.SECONDS.toNanos(timeoutInSecs);
ctx.max = max;
ctx.session = policy.createSession(cloudManager);
- List<Violation> violations = ctx.session.getViolations();
- for (Violation violation : violations) {
- violation.getClause().getThirdTag().varType.getSuggestions(ctx.setViolation(violation));
- ctx.violation = null;
- }
+ String[] t = params == null ? null : params.getParams("type");
+ List<String> types = t == null? Collections.EMPTY_LIST: Arrays.asList(t);
+
+ if(types.isEmpty() || types.contains(violation.name())) {
+ List<Violation> violations = ctx.session.getViolations();
+ for (Violation violation : violations) {
+ violation.getClause().getThirdTag().varType.getSuggestions(ctx.setViolation(violation));
+ ctx.violation = null;
+ }
- for (Violation current : ctx.session.getViolations()) {
- for (Violation old : violations) {
- if (!ctx.needMore()) return ctx.getSuggestions();
- if (current.equals(old)) {
- //could not be resolved
- ctx.suggestions.add(new Suggester.SuggestionInfo(current, null, "unresolved-violation"));
- break;
+ for (Violation current : ctx.session.getViolations()) {
+ for (Violation old : violations) {
+ if (!ctx.needMore()) return ctx.getSuggestions();
+ if (current.equals(old)) {
+ //could not be resolved
+ ctx.suggestions.add(new Suggester.SuggestionInfo(current, null, unresolved_violation));
+ break;
+ }
}
}
}
- if (ctx.needMore()) {
- try {
- addMissingReplicas(cloudManager, ctx);
- } catch (IOException e) {
- log.error("Unable to fetch cluster state", e);
+ if(types.isEmpty() || types.contains(repair.name())) {
+ if (ctx.needMore()) {
+ try {
+ addMissingReplicas(cloudManager, ctx);
+ } catch (IOException e) {
+ log.error("Unable to fetch cluster state", e);
+ }
}
}
- if (ctx.needMore()) {
- suggestOptimizations(ctx, Math.min(ctx.max - ctx.getSuggestions().size(), 10));
+ if(types.isEmpty() || types.contains(improvement.name())) {
+ if (ctx.needMore()) {
+ suggestOptimizations(ctx, Math.min(ctx.max - ctx.getSuggestions().size(), 10));
+ }
}
return ctx.getSuggestions();
}
@@ -297,7 +317,7 @@ public class PolicyHelper {
SolrRequest suggestion = ctx.addSuggestion(
ctx.session.getSuggester(ADDREPLICA)
.hint(Hint.REPLICATYPE, type)
- .hint(Hint.COLL_SHARD, new Pair(coll.getName(), shard)), "repair");
+ .hint(Hint.COLL_SHARD, new Pair(coll.getName(), shard)), Suggestion.Type.repair);
if (suggestion == null) return;
delta++;
}
@@ -323,7 +343,7 @@ public class PolicyHelper {
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
.hint(Hint.COLL_SHARD, new Pair<>(e.getKey(), shard))
.hint(Hint.SRC_NODE, row.node);
- ctx.addSuggestion(suggester, "improvement");
+ ctx.addSuggestion(suggester, Suggestion.Type.improvement);
if (ctx.getSuggestions().size() >= maxTotalSuggestions) break;
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index 3727d00..22da717 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -224,11 +224,11 @@ public abstract class Suggester implements MapWriter {
}
public static class SuggestionInfo implements MapWriter {
- String type;
+ Suggestion.Type type;
Violation violation;
SolrRequest operation;
- public SuggestionInfo(Violation violation, SolrRequest op, String type) {
+ public SuggestionInfo(Violation violation, SolrRequest op, Suggestion.Type type) {
this.violation = violation;
this.operation = op;
this.type = type;
@@ -244,7 +244,7 @@ public abstract class Suggester implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put("type", type);
+ ew.put("type", type.name());
if(violation!= null) ew.put("violation",
new ConditionalMapWriter(violation,
(k, v) -> !"violatingReplicas".equals(k)));
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 769d950..ae67e44 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -31,6 +31,11 @@ import org.apache.solr.common.util.Pair;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
public class Suggestion {
+
+ public enum Type {
+ violation, repair, improvement, unresolved_violation;
+ }
+
static class Ctx {
long endTime = -1;
int max = Integer.MAX_VALUE;
@@ -38,10 +43,10 @@ public class Suggestion {
public Violation violation;
List<Suggester.SuggestionInfo> suggestions = new ArrayList<>();
SolrRequest addSuggestion(Suggester suggester) {
- return addSuggestion(suggester, "violation");
+ return addSuggestion(suggester, Type.violation);
}
- SolrRequest addSuggestion(Suggester suggester, String type) {
+ SolrRequest addSuggestion(Suggester suggester, Type type) {
SolrRequest op = suggester.getSuggestion();
if (op != null) {
session = suggester.getSession();
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index 2d3b4dc..982b017 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -426,20 +426,20 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) getObjectByPath(m, false, "diagnostics/config"))
, cloudManagerFromDiagnostics);
for (Suggester.SuggestionInfo suggestion : suggestions) {
- assertEquals("unresolved-violation", suggestion._get("type", null));
+ assertEquals("unresolved_violation", suggestion._get("type", null));
assertEquals("1.0", suggestion._getStr("violation/violation/delta", null));
}
}
- @Ignore
+ @Ignore("This takes too long to run. enable it for perf testing")
public void testInfiniteLoop() {
Row.cacheStats.clear();
Map<String, Object> m = (Map<String, Object>) loadFromResource("testInfiniteLoop.json");
SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(
new AutoScalingConfig((Map<String, Object>) getObjectByPath(m, false, "diagnostics/config"))
- , cloudManagerFromDiagnostics, 200, 1200);
+ , cloudManagerFromDiagnostics, 200, 1200, null);
System.out.println(suggestions);
}