You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/04/16 17:21:26 UTC
[46/46] lucene-solr:jira/solr-11833: SOLR-11833: Add support for
configurable actions and metrics, and improve cold ops calculation.
SOLR-11833: Add support for configurable actions and metrics, and improve cold ops calculation.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0546c5fc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0546c5fc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0546c5fc
Branch: refs/heads/jira/solr-11833
Commit: 0546c5fcee208f9c59503e71beb196ddf5a23da8
Parents: 5bbe689
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Apr 16 19:18:43 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Apr 16 19:18:43 2018 +0200
----------------------------------------------------------------------
.../cloud/autoscaling/SearchRateTrigger.java | 140 +++++++---
.../SearchRateTriggerIntegrationTest.java | 259 ++++++++++++++++++-
.../autoscaling/DeleteReplicaSuggester.java | 4 +-
.../client/solrj/cloud/autoscaling/Policy.java | 1 +
.../solrj/cloud/autoscaling/ReplicaInfo.java | 18 ++
5 files changed, 374 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0546c5fc/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
index ecbee25..a653a14 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.cloud.autoscaling;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
@@ -34,7 +35,9 @@ import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.Pair;
@@ -50,6 +53,9 @@ import org.slf4j.LoggerFactory;
public class SearchRateTrigger extends TriggerBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String METRIC_PROP = "metric";
+ public static final String MAX_OPS_PROP = "maxOps";
+ public static final String MIN_REPLICAS_PROP = "minReplicas";
public static final String ABOVE_RATE_PROP = "aboveRate";
public static final String BELOW_RATE_PROP = "belowRate";
public static final String ABOVE_OP_PROP = "aboveOp";
@@ -66,7 +72,12 @@ public class SearchRateTrigger extends TriggerBase {
public static final String COLD_SHARDS = "coldShards";
public static final String COLD_REPLICAS = "coldReplicas";
- private String handler;
+ public static final int DEFAULT_MAX_OPS = 3;
+ public static final String DEFAULT_METRIC = "QUERY./select.requestTimes:1minRate";
+
+ private String metric;
+ private int maxOps;
+ private Integer minReplicas = null;
private String collection;
private String shard;
private String node;
@@ -85,10 +96,17 @@ public class SearchRateTrigger extends TriggerBase {
this.state.put("lastNodeEvent", lastNodeEvent);
this.state.put("lastShardEvent", lastShardEvent);
this.state.put("lastReplicaEvent", lastReplicaEvent);
- TriggerUtils.requiredProperties(requiredProperties, validProperties);
TriggerUtils.validProperties(validProperties,
AutoScalingParams.COLLECTION, AutoScalingParams.SHARD, AutoScalingParams.NODE,
- AutoScalingParams.HANDLER, ABOVE_RATE_PROP, BELOW_RATE_PROP);
+ METRIC_PROP,
+ MAX_OPS_PROP,
+ MIN_REPLICAS_PROP,
+ ABOVE_OP_PROP,
+ BELOW_OP_PROP,
+ ABOVE_NODE_OP_PROP,
+ BELOW_NODE_OP_PROP,
+ ABOVE_RATE_PROP,
+ BELOW_RATE_PROP);
}
@Override
@@ -101,7 +119,23 @@ public class SearchRateTrigger extends TriggerBase {
throw new TriggerValidationException(name, AutoScalingParams.SHARD, "When 'shard' is other than #ANY then collection name must be also other than #ANY");
}
node = (String)properties.getOrDefault(AutoScalingParams.NODE, Policy.ANY);
- handler = (String)properties.getOrDefault(AutoScalingParams.HANDLER, "/select");
+ metric = (String)properties.getOrDefault(METRIC_PROP, DEFAULT_METRIC);
+
+ String maxOpsStr = String.valueOf(properties.getOrDefault(MAX_OPS_PROP, DEFAULT_MAX_OPS));
+ try {
+ maxOps = Integer.parseInt(maxOpsStr);
+ } catch (Exception e) {
+ throw new TriggerValidationException(name, MAX_OPS_PROP, "invalid value '" + maxOpsStr + "': " + e.toString());
+ }
+
+ Object o = properties.get(MIN_REPLICAS_PROP);
+ if (o != null) {
+ try {
+ minReplicas = Integer.parseInt(o.toString());
+ } catch (Exception e) {
+ throw new TriggerValidationException(name, MIN_REPLICAS_PROP, "invalid value '" + o + "': " + e.toString());
+ }
+ }
Object above = properties.get(ABOVE_RATE_PROP);
Object below = properties.get(BELOW_RATE_PROP);
@@ -138,15 +172,21 @@ public class SearchRateTrigger extends TriggerBase {
if (belowOp == null) {
throw new TriggerValidationException(getName(), BELOW_OP_PROP, "unrecognized value: '" + belowOpStr + "'");
}
- aboveOpStr = String.valueOf(properties.getOrDefault(ABOVE_NODE_OP_PROP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()));
- belowOpStr = String.valueOf(properties.getOrDefault(BELOW_NODE_OP_PROP, CollectionParams.CollectionAction.DELETENODE.toLower()));
- aboveNodeOp = CollectionParams.CollectionAction.get(aboveOpStr);
- if (aboveNodeOp == null) {
- throw new TriggerValidationException(getName(), ABOVE_NODE_OP_PROP, "unrecognized value: '" + aboveOpStr + "'");
+ Object aboveNodeObj = properties.get(ABOVE_NODE_OP_PROP);
+ Object belowNodeObj = properties.get(BELOW_NODE_OP_PROP);
+ if (aboveNodeObj != null) {
+ try {
+ aboveNodeOp = CollectionParams.CollectionAction.get(String.valueOf(aboveNodeObj));
+ } catch (Exception e) {
+ throw new TriggerValidationException(getName(), ABOVE_NODE_OP_PROP, "unrecognized value: '" + aboveNodeObj + "'");
+ }
}
- belowNodeOp = CollectionParams.CollectionAction.get(belowOpStr);
- if (belowNodeOp == null) {
- throw new TriggerValidationException(getName(), BELOW_NODE_OP_PROP, "unrecognized value: '" + belowOpStr + "'");
+ if (belowNodeObj != null) {
+ try {
+ belowNodeOp = CollectionParams.CollectionAction.get(String.valueOf(belowNodeObj));
+ } catch (Exception e) {
+ throw new TriggerValidationException(getName(), BELOW_NODE_OP_PROP, "unrecognized value: '" + belowNodeObj + "'");
+ }
}
}
@@ -215,6 +255,13 @@ public class SearchRateTrigger extends TriggerBase {
// collection, shard, RF
Map<String, Map<String, AtomicInteger>> searchableReplicationFactors = new HashMap<>();
+ ClusterState clusterState = null;
+ try {
+ clusterState = cloudManager.getClusterStateProvider().getClusterState();
+ } catch (IOException e) {
+ log.warn("Error getting ClusterState", e);
+ return;
+ }
for (String node : cloudManager.getClusterStateProvider().getLiveNodes()) {
Map<String, ReplicaInfo> metricTags = new HashMap<>();
// coll, shard, replica
@@ -238,8 +285,7 @@ public class SearchRateTrigger extends TriggerBase {
replicaName = replica.getName(); // which is actually coreNode name...
}
String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null);
- String tag = "metrics:" + registry
- + ":QUERY." + handler + ".requestTimes:1minRate";
+ String tag = "metrics:" + registry + ":" + metric;
metricTags.put(tag, replica);
});
});
@@ -396,7 +442,7 @@ public class SearchRateTrigger extends TriggerBase {
final List<TriggerEvent.Op> ops = new ArrayList<>();
calculateHotOps(ops, searchableReplicationFactors, hotNodes, hotCollections, hotShards, hotReplicas);
- calculateColdOps(ops, searchableReplicationFactors, coldNodes, coldCollections, coldShards, coldReplicas);
+ calculateColdOps(ops, clusterState, searchableReplicationFactors, coldNodes, coldCollections, coldShards, coldReplicas);
if (ops.isEmpty()) {
return;
@@ -432,9 +478,11 @@ public class SearchRateTrigger extends TriggerBase {
// TODO: eventually we may want to commission a new node
if (!hotNodes.isEmpty() && hotShards.isEmpty() && hotCollections.isEmpty() && hotReplicas.isEmpty()) {
// move replicas around
- hotNodes.forEach((n, r) -> {
- ops.add(new TriggerEvent.Op(aboveNodeOp, Suggester.Hint.SRC_NODE, n));
- });
+ if (aboveNodeOp != null) {
+ hotNodes.forEach((n, r) -> {
+ ops.add(new TriggerEvent.Op(aboveNodeOp, Suggester.Hint.SRC_NODE, n));
+ });
+ }
} else {
// add replicas
Map<String, Map<String, List<Pair<String, String>>>> hints = new HashMap<>();
@@ -472,9 +520,9 @@ public class SearchRateTrigger extends TriggerBase {
if (numReplicas < 1) {
numReplicas = 1;
}
- // ... and at most 3 replicas
- if (numReplicas > 3) {
- numReplicas = 3;
+ // ... and at most maxOps replicas
+ if (numReplicas > maxOps) {
+ numReplicas = maxOps;
}
for (int i = 0; i < numReplicas; i++) {
hints.add(new Pair(collection, shard));
@@ -482,22 +530,27 @@ public class SearchRateTrigger extends TriggerBase {
}
private void calculateColdOps(List<TriggerEvent.Op> ops,
+ ClusterState clusterState,
Map<String, Map<String, AtomicInteger>> searchableReplicationFactors,
Map<String, Double> coldNodes,
Map<String, Double> coldCollections,
Map<String, Map<String, Double>> coldShards,
List<ReplicaInfo> coldReplicas) {
// COLD NODES:
- // Unlike in case of hot nodes, if a node is cold then any monitored
+ // Unlike the case of hot nodes, if a node is cold then any monitored
// collections / shards / replicas located on that node are cold, too.
// HOWEVER, we check only non-pull replicas and only from selected collections / shards,
// so deleting a cold node is dangerous because it may interfere with these
- // non-monitored resources
- /*
- coldNodes.forEach((node, rate) -> {
- ops.add(new TriggerEvent.Op(belowNodeOp, Suggester.Hint.SRC_NODE, node));
- });
- */
+ // non-monitored resources - this is the reason the default belowNodeOp is null / ignored.
+ //
+ // Also, note that due to the way activity is measured only nodes that contain any
+ // monitored resources are considered - there may be cold nodes in the cluster that don't
+ // belong to the monitored collections and they will be ignored.
+ if (belowNodeOp != null) {
+ coldNodes.forEach((node, rate) -> {
+ ops.add(new TriggerEvent.Op(belowNodeOp, Suggester.Hint.SRC_NODE, node));
+ });
+ }
// COLD COLLECTIONS
// Probably can't do anything reasonable about whole cold collections
@@ -509,8 +562,8 @@ public class SearchRateTrigger extends TriggerBase {
// address this by deleting cold replicas
// COLD REPLICAS:
- // Remove cold replicas but only when there's at least one more searchable replica
- // still available (additional non-searchable replicas may exist, too)
+ // Remove cold replicas but only when there's at least a minimum number of searchable
+ // replicas still available (additional non-searchable replicas may exist, too)
Map<String, Map<String, List<ReplicaInfo>>> byCollectionByShard = new HashMap<>();
coldReplicas.forEach(ri -> {
byCollectionByShard.computeIfAbsent(ri.getCollection(), c -> new HashMap<>())
@@ -519,16 +572,33 @@ public class SearchRateTrigger extends TriggerBase {
});
byCollectionByShard.forEach((coll, shards) -> {
shards.forEach((shard, replicas) -> {
- // only delete if there's at least one searchable replica left
- // again, use a simple proportional controller with a limiter
+ // only delete if there's at least minRF searchable replicas left
int rf = searchableReplicationFactors.get(coll).get(shard).get();
- if (rf > replicas.size()) {
- // delete at most 3 replicas at a time
- AtomicInteger limit = new AtomicInteger(3);
+ // we only really need a leader and we may be allowed to remove other replicas
+ int minRF = 1;
+ // but check the official RF and don't go below that
+ Integer RF = clusterState.getCollection(coll).getReplicationFactor();
+ if (RF != null) {
+ minRF = RF;
+ }
+ // unless minReplicas is set explicitly
+ if (minReplicas != null) {
+ minRF = minReplicas;
+ }
+ if (minRF < 1) {
+ minRF = 1;
+ }
+ if (rf > minRF) {
+ // delete at most maxOps replicas at a time
+ AtomicInteger limit = new AtomicInteger(Math.min(maxOps, rf - minRF));
replicas.forEach(ri -> {
if (limit.get() == 0) {
return;
}
+ // don't delete a leader
+ if (ri.getBool(ZkStateReader.LEADER_PROP, false)) {
+ return;
+ }
TriggerEvent.Op op = new TriggerEvent.Op(belowOp,
Suggester.Hint.COLL_SHARD, new Pair<>(ri.getCollection(), ri.getShard()));
op.addHint(Suggester.Hint.REPLICA, ri.getName());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0546c5fc/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
index c1412ab..370b23a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
@@ -22,10 +22,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.util.concurrent.AtomicDouble;
import org.apache.lucene.util.LuceneTestCase;
@@ -37,12 +36,16 @@ import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.CloudTestUtils;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.LogLevel;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@@ -56,16 +59,16 @@ import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.timeSourc
* Integration test for {@link SearchRateTrigger}
*/
@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028")
+@LuceneTestCase.Slow
public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
private static CountDownLatch listenerCreated = new CountDownLatch(1);
- private static int waitForSeconds = 1;
- private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
private static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
- static CountDownLatch finished = new CountDownLatch(1);
+ private static CountDownLatch finished = new CountDownLatch(1);
+ private static SolrCloudManager cloudManager;
+
+ private int waitForSeconds;
@BeforeClass
public static void setupCluster() throws Exception {
@@ -80,21 +83,36 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
SolrClient solrClient = cluster.getSolrClient();
NamedList<Object> response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
+ cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
+ }
+
+ @Before
+ public void beforeTest() throws Exception {
+ cluster.deleteAllCollections();
+ finished = new CountDownLatch(1);
+ listenerEvents.clear();
+ waitForSeconds = 3 + random().nextInt(5);
}
@Test
public void testAboveSearchRate() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
- String COLL1 = "collection1";
+ String COLL1 = "aboveRate_collection";
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
"conf", 1, 2);
create.process(solrClient);
+
+ CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 2));
+
+ // the trigger is initially disabled so that we have the time to set up listeners
+ // and generate the traffic
String setTriggerCommand = "{" +
"'set-trigger' : {" +
"'name' : 'search_rate_trigger'," +
"'event' : 'searchRate'," +
"'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
+ "'enabled' : false," +
"'collection' : '" + COLL1 + "'," +
"'aboveRate' : 1.0," +
"'belowRate' : 0.1," +
@@ -134,13 +152,23 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
-
SolrParams query = params(CommonParams.Q, "*:*");
for (int i = 0; i < 500; i++) {
solrClient.query(COLL1, query);
}
+ // enable the trigger
+ String resumeTriggerCommand = "{" +
+ "'resume-trigger' : {" +
+ "'name' : 'search_rate_trigger'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
boolean await = finished.await(20, TimeUnit.SECONDS);
assertTrue("The trigger did not fire at all", await);
@@ -201,7 +229,216 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
@Test
public void testBelowSearchRate() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String COLL1 = "belowRate_collection";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
+ "conf", 1, 2);
+ create.process(solrClient);
+ // add a couple of spare replicas above RF. Use different types to verify that only
+ // searchable replicas are considered
+ // these additional replicas will be placed on other nodes in the cluster
+ solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.NRT));
+ solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.TLOG));
+ solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.PULL));
+
+ CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 5));
+
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'search_rate_trigger'," +
+ "'event' : 'searchRate'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : false," +
+ "'collection' : '" + COLL1 + "'," +
+ "'aboveRate' : 1.0," +
+ "'belowRate' : 0.1," +
+ "'belowNodeOp' : 'none'," +
+ "'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'srt'," +
+ "'trigger' : 'search_rate_trigger'," +
+ "'stage' : ['FAILED','SUCCEEDED']," +
+ "'afterAction': ['compute', 'execute']," +
+ "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'finished'," +
+ "'trigger' : 'search_rate_trigger'," +
+ "'stage' : ['SUCCEEDED']," +
+ "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
+ // enable the trigger
+ String resumeTriggerCommand = "{" +
+ "'resume-trigger' : {" +
+ "'name' : 'search_rate_trigger'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
+ boolean await = finished.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+
+ // suspend the trigger
+ // enable the trigger
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : 'search_rate_trigger'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(5000);
+
+ List<CapturedEvent> events = listenerEvents.get("srt");
+ assertEquals(events.toString(), 3, events.size());
+ CapturedEvent ev = events.get(0);
+ assertEquals(ev.toString(), "compute", ev.actionName);
+ List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
+ assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
+ // 3 cold nodes, 2 cold replicas
+ assertEquals(ops.toString(), 5, ops.size());
+ AtomicInteger coldNodes = new AtomicInteger();
+ AtomicInteger coldReplicas = new AtomicInteger();
+ ops.forEach(op -> {
+ if (op.getAction().equals(CollectionParams.CollectionAction.NONE)) {
+ coldNodes.incrementAndGet();
+ } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) {
+ coldReplicas.incrementAndGet();
+ } else {
+ fail("unexpected op: " + op);
+ }
+ });
+ assertEquals("cold nodes", 3, coldNodes.get());
+ assertEquals("cold replicas", 2, coldReplicas.get());
+
+ // now the collection should be back to RF = 2, with one additional PULL replica
+ CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 3));
+
+ listenerEvents.clear();
+ finished = new CountDownLatch(1);
+
+ // resume trigger
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // there should be only coldNode ops now, and no coldReplica ops since searchable RF == collection RF
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
+ await = finished.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+
+ // suspend trigger
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(5000);
+
+ events = listenerEvents.get("srt");
+ assertEquals(events.toString(), 3, events.size());
+
+ ev = events.get(0);
+ assertEquals(ev.toString(), "compute", ev.actionName);
+ ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
+ assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
+ assertEquals(ops.toString(), 1, ops.size());
+ assertEquals(ops.toString(), CollectionParams.CollectionAction.NONE, ops.get(0).getAction());
+
+ listenerEvents.clear();
+ finished = new CountDownLatch(1);
+
+ // now allow single replicas
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'search_rate_trigger'," +
+ "'event' : 'searchRate'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'collection' : '" + COLL1 + "'," +
+ "'aboveRate' : 1.0," +
+ "'belowRate' : 0.1," +
+ "'minReplicas' : 1," +
+ "'belowNodeOp' : 'none'," +
+ "'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
+ await = finished.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+
+ // suspend trigger
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(5000);
+
+ events = listenerEvents.get("srt");
+ assertEquals(events.toString(), 3, events.size());
+
+ ev = events.get(0);
+ assertEquals(ev.toString(), "compute", ev.actionName);
+ ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
+ assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
+ assertEquals(ops.toString(), 2, ops.size());
+ AtomicInteger coldNodes2 = new AtomicInteger();
+ AtomicInteger coldReplicas2 = new AtomicInteger();
+ ops.forEach(op -> {
+ if (op.getAction().equals(CollectionParams.CollectionAction.NONE)) {
+ coldNodes2.incrementAndGet();
+ } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) {
+ coldReplicas2.incrementAndGet();
+ } else {
+ fail("unexpected op: " + op);
+ }
+ });
+
+ assertEquals("coldNodes", 1, coldNodes2.get());
+ assertEquals("colReplicas", 1, coldReplicas2.get());
+ // now the collection should be at RF == 1, with one additional PULL replica
+ CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 2));
}
public static class CapturingTriggerListener extends TriggerListenerBase {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0546c5fc/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java
index a7d5d70..9a942ad 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java
@@ -25,8 +25,8 @@ import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.Pair;
/**
- * This suggester produces a DELETEREPLICA request using provided {@link Hint#COLL_SHARD} and
- * {@link Hint#NUMBER} hints to specify the collection, shard and number of replicas to delete.
+ * This suggester produces a DELETEREPLICA request using provided {@link org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint#COLL_SHARD} and
+ * {@link org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint#NUMBER} hints to specify the collection, shard and number of replicas to delete.
*/
class DeleteReplicaSuggester extends Suggester {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0546c5fc/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 74a4d1f..2f729d9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -470,6 +470,7 @@ public class Policy implements MapWriter {
ops.put(CollectionAction.MOVEREPLICA, () -> new MoveReplicaSuggester());
ops.put(CollectionAction.SPLITSHARD, () -> new SplitShardSuggester());
ops.put(CollectionAction.MERGESHARDS, () -> new UnsupportedSuggester(CollectionAction.MERGESHARDS));
+ ops.put(CollectionAction.NONE, () -> new UnsupportedSuggester(CollectionAction.NONE));
}
public Map<String, List<Clause>> getPolicies() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0546c5fc/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
index e1d8281..50e77f8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
@@ -138,6 +138,24 @@ public class ReplicaInfo implements MapWriter {
return variables.get(name);
}
+ public Object getVariable(String name, Object defValue) {
+ Object o = variables.get(name);
+ if (o != null) {
+ return o;
+ } else {
+ return defValue;
+ }
+ }
+
+ public boolean getBool(String name, boolean defValue) {
+ Object o = getVariable(name, defValue);
+ if (o instanceof Boolean) {
+ return (Boolean)o;
+ } else {
+ return Boolean.parseBoolean(String.valueOf(o));
+ }
+ }
+
@Override
public String toString() {
return Utils.toJSONString(this);