You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2020/06/03 19:13:47 UTC
[lucene-solr] 18/47: SOLR-13325: Add a collection selector to
ComputePlanAction (#1512)
This is an automated email from the ASF dual-hosted git repository.
sokolov pushed a commit to branch jira/lucene-8962
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit a00d7eb4399c2e90182a4deeac28e015a0eaac92
Author: Shalin Shekhar Mangar <sh...@apache.org>
AuthorDate: Fri May 22 10:36:27 2020 +0530
SOLR-13325: Add a collection selector to ComputePlanAction (#1512)
ComputePlanAction now supports a collection selector of the form `collections: {policy: my_policy}` which can be used to select multiple collections that match collection property/value pairs. This is useful to maintain a whitelist of collections for which actions should be taken without needing to hard-code the collection names. The collection hints are pushed down to the policy engine so operations for non-matching collections are not computed at all. The AutoAddReplicasPlanAction n [...]
---
solr/CHANGES.txt | 6 +
.../autoscaling/AutoAddReplicasPlanAction.java | 47 ++----
.../solr/cloud/autoscaling/ComputePlanAction.java | 167 +++++++++++++--------
.../autoscaling/AutoAddReplicasPlanActionTest.java | 12 +-
.../cloud/autoscaling/ComputePlanActionTest.java | 80 ++++++----
.../src/solrcloud-autoscaling-trigger-actions.adoc | 71 ++++++++-
6 files changed, 245 insertions(+), 138 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 2629407..51c0fec 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -119,6 +119,12 @@ Improvements
* SOLR-14407: Handle shards.purpose in the postlogs tool (Joel Bernstein)
+* SOLR-13325: ComputePlanAction now supports a collection selector of the form `collections: {policy: my_policy}`
+ which can be used to select multiple collections that match collection property/value pairs. This is useful to
+ maintain a whitelist of collections for which actions are taken without needing to hard code the collection names
+ themselves. The collection hints are pushed down to the policy engine so operations for non-matching collections
+ are not computed at all. (ab, shalin)
+
Optimizations
---------------------
* SOLR-8306: Do not collect expand documents when expand.rows=0 (Marshall Sanders, Amelia Henderson)
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanAction.java
index fdd3474..d129fdb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanAction.java
@@ -18,46 +18,23 @@
package org.apache.solr.cloud.autoscaling;
-import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
-import org.apache.solr.client.solrj.cloud.autoscaling.NoneSuggester;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
-import org.apache.solr.client.solrj.impl.ClusterStateProvider;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.SolrResourceLoader;
+import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
+
+/**
+ * This class configures the parent ComputePlanAction to compute plan
+ * only for collections which have autoAddReplicas=true.
+ */
public class AutoAddReplicasPlanAction extends ComputePlanAction {
@Override
- protected Suggester getSuggester(Policy.Session session, TriggerEvent event, ActionContext context, SolrCloudManager cloudManager) throws IOException {
- // for backward compatibility
- ClusterStateProvider stateProvider = cloudManager.getClusterStateProvider();
- String autoAddReplicas = stateProvider.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, (String) null);
- if (autoAddReplicas != null && autoAddReplicas.equals("false")) {
- return NoneSuggester.get(session);
- }
-
- Suggester suggester = super.getSuggester(session, event, context, cloudManager);
- ClusterState clusterState;
- try {
- clusterState = stateProvider.getClusterState();
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Exception getting cluster state", e);
- }
-
- boolean anyCollections = false;
- for (DocCollection collection: clusterState.getCollectionsMap().values()) {
- if (collection.getAutoAddReplicas()) {
- anyCollections = true;
- suggester.hint(Suggester.Hint.COLL, collection.getName());
- }
- }
-
- if (!anyCollections) return NoneSuggester.get(session);
- return suggester;
+ public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map<String, Object> properties) throws TriggerValidationException {
+ properties.put("collections", Collections.singletonMap(AUTO_ADD_REPLICAS, "true"));
+ super.configure(loader, cloudManager, properties);
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 87dd0c3..fad45e0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -17,38 +17,28 @@
package org.apache.solr.cloud.autoscaling;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.NoneSuggester;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
-import org.apache.solr.client.solrj.cloud.autoscaling.UnsupportedSuggester;
+import org.apache.solr.client.solrj.cloud.autoscaling.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
import static org.apache.solr.cloud.autoscaling.TriggerEvent.NODE_NAMES;
/**
@@ -61,7 +51,8 @@ import static org.apache.solr.cloud.autoscaling.TriggerEvent.NODE_NAMES;
public class ComputePlanAction extends TriggerActionBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- Set<String> collections = new HashSet<>();
+ // accept all collections by default
+ Predicate<String> collectionsPredicate = s -> true;
public ComputePlanAction() {
super();
@@ -72,9 +63,37 @@ public class ComputePlanAction extends TriggerActionBase {
@Override
public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map<String, Object> properties) throws TriggerValidationException {
super.configure(loader, cloudManager, properties);
- String colString = (String) properties.get("collections");
- if (colString != null && !colString.isEmpty()) {
- collections.addAll(StrUtils.splitSmart(colString, ','));
+
+ Object value = properties.get("collections");
+ if (value instanceof String) {
+ String colString = (String) value;
+ if (!colString.isEmpty()) {
+ List<String> whiteListedCollections = StrUtils.splitSmart(colString, ',');
+ collectionsPredicate = whiteListedCollections::contains;
+ }
+ } else if (value instanceof Map) {
+ Map<String, String> matchConditions = (Map<String, String>) value;
+ collectionsPredicate = collectionName -> {
+ try {
+ DocCollection collection = cloudManager.getClusterStateProvider().getCollection(collectionName);
+ if (collection == null) {
+ log.debug("Collection: {} was not found while evaluating conditions", collectionName);
+ return false;
+ }
+ for (Map.Entry<String, String> entry : matchConditions.entrySet()) {
+ if (!entry.getValue().equals(collection.get(entry.getKey()))) {
+ if (log.isDebugEnabled()) {
+ log.debug("Collection: {} does not match condition: {}:{}", collectionName, entry.getKey(), entry.getValue());
+ }
+ return false;
+ }
+ }
+ return true;
+ } catch (IOException e) {
+ log.error("Exception fetching collection information for: {}", collectionName, e);
+ return false;
+ }
+ };
}
}
@@ -142,14 +161,6 @@ public class ComputePlanAction extends TriggerActionBase {
if (log.isDebugEnabled()) {
log.debug("Computed Plan: {}", operation.getParams());
}
- if (!collections.isEmpty()) {
- String coll = operation.getParams().get(CoreAdminParams.COLLECTION);
- if (coll != null && !collections.contains(coll)) {
- // discard an op that doesn't affect our collections
- log.debug("-- discarding due to collection={} not in {}", coll, collections);
- continue;
- }
- }
Map<String, Object> props = context.getProperties();
props.compute("operations", (k, v) -> {
List<SolrRequest> operations = (List<SolrRequest>) v;
@@ -217,29 +228,7 @@ public class ComputePlanAction extends TriggerActionBase {
suggester = getNodeAddedSuggester(cloudManager, session, event);
break;
case NODELOST:
- String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
- CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);
- switch (action) {
- case MOVEREPLICA:
- suggester = session.getSuggester(action)
- .hint(Suggester.Hint.SRC_NODE, event.getProperty(NODE_NAMES));
- break;
- case DELETENODE:
- int start = (Integer)event.getProperty(START, 0);
- List<String> srcNodes = (List<String>) event.getProperty(NODE_NAMES);
- if (srcNodes.isEmpty() || start >= srcNodes.size()) {
- return NoneSuggester.get(session);
- }
- String sourceNode = srcNodes.get(start);
- suggester = session.getSuggester(action)
- .hint(Suggester.Hint.SRC_NODE, Collections.singletonList(sourceNode));
- event.getProperties().put(START, ++start);
- break;
- case NONE:
- return NoneSuggester.get(session);
- default:
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unsupported preferredOperation: " + action.toLower() + " specified for node lost trigger");
- }
+ suggester = getNodeLostSuggester(cloudManager, session, event);
break;
case SEARCHRATE:
case METRIC:
@@ -258,13 +247,15 @@ public class ComputePlanAction extends TriggerActionBase {
for (Map.Entry<Suggester.Hint, Object> e : op.getHints().entrySet()) {
suggester = suggester.hint(e.getKey(), e.getValue());
}
+ if (applyCollectionHints(cloudManager, suggester) == 0) return NoneSuggester.get(session);
suggester = suggester.forceOperation(true);
event.getProperties().put(START, ++start);
break;
case SCHEDULED:
- preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
- action = CollectionParams.CollectionAction.get(preferredOp);
+ String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
+ CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);
suggester = session.getSuggester(action);
+ if (applyCollectionHints(cloudManager, suggester) == 0) return NoneSuggester.get(session);
break;
default:
throw new UnsupportedOperationException("No support for events other than nodeAdded, nodeLost, searchRate, metric, scheduled and indexSize. Received: " + event.getEventType());
@@ -272,6 +263,53 @@ public class ComputePlanAction extends TriggerActionBase {
return suggester;
}
+ private Suggester getNodeLostSuggester(SolrCloudManager cloudManager, Policy.Session session, TriggerEvent event) throws IOException {
+ String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
+ CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);
+ switch (action) {
+ case MOVEREPLICA:
+ Suggester s = session.getSuggester(action)
+ .hint(Suggester.Hint.SRC_NODE, event.getProperty(NODE_NAMES));
+ if (applyCollectionHints(cloudManager, s) == 0) return NoneSuggester.get(session);
+ return s;
+ case DELETENODE:
+ int start = (Integer)event.getProperty(START, 0);
+ List<String> srcNodes = (List<String>) event.getProperty(NODE_NAMES);
+ if (srcNodes.isEmpty() || start >= srcNodes.size()) {
+ return NoneSuggester.get(session);
+ }
+ String sourceNode = srcNodes.get(start);
+ s = session.getSuggester(action)
+ .hint(Suggester.Hint.SRC_NODE, event.getProperty(NODE_NAMES));
+ if (applyCollectionHints(cloudManager, s) == 0) return NoneSuggester.get(session);
+ s.hint(Suggester.Hint.SRC_NODE, Collections.singletonList(sourceNode));
+ event.getProperties().put(START, ++start);
+ return s;
+ case NONE:
+ return NoneSuggester.get(session);
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unsupported preferredOperation: " + action.toLower() + " specified for node lost trigger");
+ }
+ }
+
+ /**
+ * Applies collection hints for all collections that match the {@link #collectionsPredicate}
+ * and returns the number of collections that matched.
+ * @return number of collections that match the {@link #collectionsPredicate}
+ * @throws IOException if {@link org.apache.solr.client.solrj.impl.ClusterStateProvider} throws IOException
+ */
+ private int applyCollectionHints(SolrCloudManager cloudManager, Suggester s) throws IOException {
+ ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
+ Set<String> set = clusterState.getCollectionStates().keySet().stream()
+ .filter(collectionRef -> collectionsPredicate.test(collectionRef))
+ .collect(Collectors.toSet());
+ if (set.size() < clusterState.getCollectionStates().size()) {
+ // apply hints only if a subset of collections are selected
+ set.forEach(c -> s.hint(Suggester.Hint.COLL, c));
+ }
+ return set.size();
+ }
+
private Suggester getNodeAddedSuggester(SolrCloudManager cloudManager, Policy.Session session, TriggerEvent event) throws IOException {
String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
Replica.Type replicaType = (Replica.Type) event.getProperty(AutoScalingParams.REPLICA_TYPE, Replica.Type.NRT);
@@ -283,17 +321,18 @@ public class ComputePlanAction extends TriggerActionBase {
case ADDREPLICA:
// add all collection/shard pairs and let policy engine figure out which one
// to place on the target node
- // todo in future we can prune ineligible collection/shard pairs
ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
Set<Pair<String, String>> collShards = new HashSet<>();
- clusterState.getCollectionStates().forEach((collectionName, collectionRef) -> {
- DocCollection docCollection = collectionRef.get();
- if (docCollection != null) {
- docCollection.getActiveSlices().stream()
- .map(slice -> new Pair<>(collectionName, slice.getName()))
- .forEach(collShards::add);
- }
- });
+ clusterState.getCollectionStates().entrySet().stream()
+ .filter(e -> collectionsPredicate.test(e.getKey()))
+ .forEach(entry -> {
+ DocCollection docCollection = entry.getValue().get();
+ if (docCollection != null) {
+ docCollection.getActiveSlices().stream()
+ .map(slice -> new Pair<>(entry.getKey(), slice.getName()))
+ .forEach(collShards::add);
+ }
+ });
suggester.hint(Suggester.Hint.COLL_SHARD, collShards);
suggester.hint(Suggester.Hint.REPLICATYPE, replicaType);
break;
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
index b6e6d20..8b41f2f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
@@ -84,6 +84,7 @@ public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
String collection1 = "testSimple1";
String collection2 = "testSimple2";
+ String collection3 = "testSimple3";
CollectionAdminRequest.createCollection(collection1, "conf", 2, 2)
.setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
.setAutoAddReplicas(true)
@@ -94,8 +95,8 @@ public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
.setAutoAddReplicas(false)
.setMaxShardsPerNode(1)
.process(cluster.getSolrClient());
- // the number of cores in jetty1 (5) will be larger than jetty3 (1)
- CollectionAdminRequest.createCollection("testSimple3", "conf", 3, 1)
+ // the number of cores in jetty1 (6) will be larger than jetty3 (1)
+ CollectionAdminRequest.createCollection(collection3, "conf", 3, 1)
.setCreateNodeSet(jetty1.getNodeName())
.setAutoAddReplicas(false)
.setMaxShardsPerNode(3)
@@ -103,7 +104,7 @@ public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
cluster.waitForActiveCollection(collection1, 2, 4);
cluster.waitForActiveCollection(collection2, 1, 2);
- cluster.waitForActiveCollection("testSimple3", 3, 3);
+ cluster.waitForActiveCollection(collection3, 3, 3);
// we remove the implicit created trigger, so the replicas won't be moved
String removeTriggerCommand = "{" +
@@ -139,7 +140,7 @@ public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
cluster.waitForActiveCollection(collection1, 2, 4);
cluster.waitForActiveCollection(collection2, 1, 2);
- cluster.waitForActiveCollection("testSimple3", 3, 3);
+ cluster.waitForActiveCollection(collection3, 3, 3);
assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 30000));
@@ -184,7 +185,7 @@ public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
cluster.waitForActiveCollection(collection1, 2, 4);
cluster.waitForActiveCollection(collection2, 1, 2);
- cluster.waitForActiveCollection("testSimple3", 3, 3);
+ cluster.waitForActiveCollection(collection3, 3, 3);
assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 30000));
@@ -211,6 +212,7 @@ public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
@SuppressForbidden(reason = "Needs currentTimeMillis to create unique id")
private List<SolrRequest> getOperations(JettySolrRunner actionJetty, String lostNodeName) throws Exception {
try (AutoAddReplicasPlanAction action = new AutoAddReplicasPlanAction()) {
+ action.configure(actionJetty.getCoreContainer().getResourceLoader(), actionJetty.getCoreContainer().getZkController().getSolrCloudManager(), new HashMap<>());
TriggerEvent lostNode = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST, ".auto_add_replicas", Collections.singletonList(System.currentTimeMillis()), Collections.singletonList(lostNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
ActionContext context = new ActionContext(actionJetty.getCoreContainer().getZkController().getSolrCloudManager(), null, new HashMap<>());
action.process(lostNode, context);
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 1f5c8e3..471a0a7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -17,19 +17,6 @@
package org.apache.solr.cloud.autoscaling;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
@@ -55,14 +42,18 @@ import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.LogLevel;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -444,7 +435,33 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
@Test
//2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018
- public void testSelectedCollections() throws Exception {
+ public void testSelectedCollectionsByName() throws Exception {
+ String collectionsFilter = "'testSelected1,testSelected2'";
+ testCollectionsPredicate(collectionsFilter, Collections.emptyMap());
+ }
+
+ @Test
+ //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018
+ public void testSelectedCollectionsByPolicy() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setSearchPolicyCommand = "{" +
+ " 'set-policy': {" +
+ " 'search': [" +
+ " {'replica':'<5', 'shard': '#EACH', 'node': '#ANY'}," +
+ " ]" +
+ "}}";
+ SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setSearchPolicyCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String collectionsFilter = "{'policy': 'search'}";
+ Map<String, String> createCollectionParameters = new HashMap<>();
+ createCollectionParameters.put("testSelected1", "search");
+ createCollectionParameters.put("testSelected2", "search");
+ testCollectionsPredicate(collectionsFilter, createCollectionParameters);
+ }
+
+ private void testCollectionsPredicate(String collectionsFilter, Map<String, String> createCollectionParameters) throws Exception {
if (log.isInfoEnabled()) {
log.info("Found number of jetties: {}", cluster.getJettySolrRunners().size());
}
@@ -457,28 +474,37 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
CloudSolrClient solrClient = cluster.getSolrClient();
String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction', 'collections' : 'testSelected1,testSelected2'}," +
- "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '1s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction', 'collections' : " + collectionsFilter + "}," +
+ "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" +
+ "}}";
SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
NamedList<Object> response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testSelected1",
"conf", 2, 2);
+ if (createCollectionParameters.get("testSelected1") != null) {
+ create.setPolicy(createCollectionParameters.get("testSelected1"));
+ }
create.process(solrClient);
create = CollectionAdminRequest.createCollection("testSelected2",
"conf", 2, 2);
+ if (createCollectionParameters.get("testSelected2") != null) {
+ create.setPolicy(createCollectionParameters.get("testSelected2"));
+ }
create.process(solrClient);
create = CollectionAdminRequest.createCollection("testSelected3",
"conf", 2, 2);
+ if (createCollectionParameters.get("testSelected3") != null) {
+ create.setPolicy(createCollectionParameters.get("testSelected3"));
+ }
create.process(solrClient);
cluster.waitForActiveCollection("testSelected1", 2, 4);
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-trigger-actions.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-trigger-actions.adoc
index 3ad3772..099f992 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-trigger-actions.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-trigger-actions.adoc
@@ -29,12 +29,19 @@ commands which can re-balance the cluster in response to trigger events.
The following parameters are configurable:
`collections`::
-A comma-separated list of collection names. If this list is not empty then
-the computed operations will only calculate collection operations that affect
-listed collections and ignore any other collection operations for collections
-not listed here. Note that non-collection operations are not affected by this.
+A comma-separated list of collection names, or a selector on collection properties that can be used to filter collections for which the plan is computed.
-Example configuration:
+If a non-empty list or selector is specified then the computed operations will only calculate collection operations that affect
+matched collections and ignore any other collection operations for collections
+not listed here. This does not affect non-collection operations.
+
+A collection selector is of the form `collections: {key1: value1, key2: value2, ...}` where the key can be any collection property such as `name`, `policy`, `numShards` etc.
+The value must match exactly and all specified properties must match for a collection to match.
+
+A collection selector is useful in a cluster where collections are added and removed frequently and where selecting only collections that
+use a specific autoscaling policy is useful.
+
+Example configurations:
[source,json]
----
@@ -48,11 +55,11 @@ Example configuration:
{
"name" : "compute_plan",
"class" : "solr.ComputePlanAction",
- "collections" : "test1,test2",
+ "collections" : "test1,test2"
},
{
"name" : "execute_plan",
- "class" : "solr.ExecutePlanAction",
+ "class" : "solr.ExecutePlanAction"
}
]
}
@@ -63,6 +70,56 @@ In this example only collections `test1` and `test2` will be potentially
replicated / moved to an added node, other collections will be ignored even
if they cause policy violations.
+[source,json]
+----
+{
+ "set-trigger" : {
+ "name" : "node_added_trigger",
+ "event" : "nodeAdded",
+ "waitFor" : "1s",
+ "enabled" : true,
+ "actions" : [
+ {
+ "name" : "compute_plan",
+ "class" : "solr.ComputePlanAction",
+ "collections" : {"policy": "my_policy"}
+ },
+ {
+ "name" : "execute_plan",
+ "class" : "solr.ExecutePlanAction"
+ }
+ ]
+ }
+}
+----
+
+In this example only collections which use the `my_policy` as their autoscaling policy will be potentially replicated / moved to an added node, other collections will be ignored even if they cause policy violations.
+
+[source,json]
+----
+{
+ "set-trigger" : {
+ "name" : "node_added_trigger",
+ "event" : "nodeAdded",
+ "waitFor" : "1s",
+ "enabled" : true,
+ "actions" : [
+ {
+ "name" : "compute_plan",
+ "class" : "solr.ComputePlanAction",
+ "collections" : {"policy": "my_policy", "numShards" : "4"}
+ },
+ {
+ "name" : "execute_plan",
+ "class" : "solr.ExecutePlanAction"
+ }
+ ]
+ }
+}
+----
+
+In this example only collections which use the `my_policy` as their autoscaling policy and that have `numShards` equal to `4` will be potentially replicated / moved to an added node, other collections will be ignored even if they cause policy violations.
+
== Execute Plan Action
The `ExecutePlanAction` executes the Collection API commands emitted by the `ComputePlanAction` against