You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/04/17 17:06:24 UTC
lucene-solr:jira/solr-11833: SOLR-11833: Reverse the order of cold
node / cold replica requests. Add a test for DELETENODE condition.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11833 f4c314012 -> 516aad5a5
SOLR-11833: Reverse the order of cold node / cold replica requests. Add a test for
DELETENODE condition.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/516aad5a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/516aad5a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/516aad5a
Branch: refs/heads/jira/solr-11833
Commit: 516aad5a52795020faa2f43bfe4ae828a9253ed9
Parents: f4c3140
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Apr 17 19:05:43 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Apr 17 19:05:43 2018 +0200
----------------------------------------------------------------------
.../cloud/autoscaling/SearchRateTrigger.java | 36 +--
.../SearchRateTriggerIntegrationTest.java | 231 ++++++++++++++++++-
.../src/solrcloud-autoscaling-triggers.adoc | 10 +-
3 files changed, 250 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/516aad5a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
index 7722b00..238fcde 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
@@ -545,22 +545,6 @@ public class SearchRateTrigger extends TriggerBase {
Map<String, Double> coldCollections,
Map<String, Map<String, Double>> coldShards,
List<ReplicaInfo> coldReplicas) {
- // COLD NODES:
- // Unlike the case of hot nodes, if a node is cold then any monitored
- // collections / shards / replicas located on that node are cold, too.
- // HOWEVER, we check only non-pull replicas and only from selected collections / shards,
- // so deleting a cold node is dangerous because it may interfere with these
- // non-monitored resources - this is the reason the default belowNodeOp is null / ignored.
- //
- // Also, note that due to the way activity is measured only nodes that contain any
- // monitored resources are considered - there may be cold nodes in the cluster that don't
- // belong to the monitored collections and they will be ignored.
- if (belowNodeOp != null) {
- coldNodes.forEach((node, rate) -> {
- ops.add(new TriggerEvent.Op(belowNodeOp, Suggester.Hint.SRC_NODE, node));
- });
- }
-
// COLD COLLECTIONS
// Probably can't do anything reasonable about whole cold collections
// because they may be needed even if not used.
@@ -573,6 +557,8 @@ public class SearchRateTrigger extends TriggerBase {
// COLD REPLICAS:
// Remove cold replicas but only when there's at least a minimum number of searchable
// replicas still available (additional non-searchable replicas may exist, too)
+ // NOTE: do this before adding ops for DELETENODE because we don't want to attempt
+ // deleting replicas that have been already moved elsewhere
Map<String, Map<String, List<ReplicaInfo>>> byCollectionByShard = new HashMap<>();
coldReplicas.forEach(ri -> {
byCollectionByShard.computeIfAbsent(ri.getCollection(), c -> new HashMap<>())
@@ -617,6 +603,24 @@ public class SearchRateTrigger extends TriggerBase {
}
});
});
+
+ // COLD NODES:
+ // Unlike the case of hot nodes, if a node is cold then any monitored
+ // collections / shards / replicas located on that node are cold, too.
+ // HOWEVER, we check only non-pull replicas and only from selected collections / shards,
+ // so deleting a cold node is dangerous because it may interfere with these
+ // non-monitored resources - this is the reason the default belowNodeOp is null / ignored.
+ //
+ // Also, note that due to the way activity is measured only nodes that contain any
+ // monitored resources are considered - there may be cold nodes in the cluster that don't
+ // belong to the monitored collections and they will be ignored.
+ if (belowNodeOp != null) {
+ coldNodes.forEach((node, rate) -> {
+ ops.add(new TriggerEvent.Op(belowNodeOp, Suggester.Hint.SRC_NODE, node));
+ });
+ }
+
+
}
private boolean waitForElapsed(String name, long now, Map<String, Long> lastEventMap) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/516aad5a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
index 365e6b5..86b7f5f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
@@ -71,6 +71,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
private static CountDownLatch listenerCreated = new CountDownLatch(1);
private static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
private static CountDownLatch finished = new CountDownLatch(1);
+ private static CountDownLatch started = new CountDownLatch(1);
private static SolrCloudManager cloudManager;
private int waitForSeconds;
@@ -104,6 +105,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
finished = new CountDownLatch(1);
+ started = new CountDownLatch(1);
listenerEvents = new HashMap<>();
waitForSeconds = 3 + random().nextInt(5);
}
@@ -120,7 +122,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
"conf", 1, 2);
create.process(solrClient);
- CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
CloudTestUtils.clusterShape(1, 2));
// the trigger is initially disabled so that we have the time to set up listeners
@@ -146,6 +148,19 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
String setListenerCommand = "{" +
"'set-listener' : " +
"{" +
+ "'name' : 'started'," +
+ "'trigger' : 'search_rate_trigger1'," +
+ "'stage' : ['STARTED']," +
+ "'class' : '" + StartedProcessingListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
"'name' : 'srt'," +
"'trigger' : 'search_rate_trigger1'," +
"'stage' : ['FAILED','SUCCEEDED']," +
@@ -187,9 +202,12 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
- boolean await = finished.await(20, TimeUnit.SECONDS);
+ boolean await = started.await(20, TimeUnit.SECONDS);
assertTrue("The trigger did not fire at all", await);
+ await = finished.await(60, TimeUnit.SECONDS);
+ assertTrue("The trigger did not finish processing", await);
+
// suspend the trigger
String suspendTriggerCommand = "{" +
"'suspend-trigger' : {" +
@@ -262,7 +280,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
"conf", 1, 2);
create.process(solrClient);
- CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
CloudTestUtils.clusterShape(1, 2));
// add a couple of spare replicas above RF. Use different types to verify that only
@@ -272,7 +290,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.TLOG));
solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.PULL));
- CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
CloudTestUtils.clusterShape(1, 5));
String setTriggerCommand = "{" +
@@ -298,6 +316,19 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
String setListenerCommand = "{" +
"'set-listener' : " +
"{" +
+ "'name' : 'started'," +
+ "'trigger' : 'search_rate_trigger2'," +
+ "'stage' : ['STARTED']," +
+ "'class' : '" + StartedProcessingListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
"'name' : 'srt'," +
"'trigger' : 'search_rate_trigger2'," +
"'stage' : ['FAILED','SUCCEEDED']," +
@@ -336,8 +367,10 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
- boolean await = finished.await(20, TimeUnit.SECONDS);
+ boolean await = started.await(20, TimeUnit.SECONDS);
assertTrue("The trigger did not fire at all", await);
+ await = finished.await(60, TimeUnit.SECONDS);
+ assertTrue("The trigger did not finish processing", await);
// suspend the trigger
String suspendTriggerCommand = "{" +
@@ -374,11 +407,12 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
assertEquals("cold replicas", 2, coldReplicas.get());
// now the collection should be back to RF = 2, with one additional PULL replica
- CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
CloudTestUtils.clusterShape(1, 3));
listenerEvents.clear();
finished = new CountDownLatch(1);
+ started = new CountDownLatch(1);
// resume trigger
req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
@@ -388,8 +422,10 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
// there should be only coldNode ops now, and no coldReplica ops since searchable RF == collection RF
timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
- await = finished.await(20, TimeUnit.SECONDS);
+ await = started.await(20, TimeUnit.SECONDS);
assertTrue("The trigger did not fire at all", await);
+ await = finished.await(60, TimeUnit.SECONDS);
+ assertTrue("The trigger did not finish processing", await);
// suspend trigger
req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
@@ -410,6 +446,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
listenerEvents.clear();
finished = new CountDownLatch(1);
+ started = new CountDownLatch(1);
// now allow single replicas
setTriggerCommand = "{" +
@@ -434,8 +471,10 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
- await = finished.await(20, TimeUnit.SECONDS);
+ await = started.await(20, TimeUnit.SECONDS);
assertTrue("The trigger did not fire at all", await);
+ await = finished.await(60, TimeUnit.SECONDS);
+ assertTrue("The trigger did not finish processing", await);
// suspend trigger
req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
@@ -468,7 +507,173 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
assertEquals("colReplicas", 1, coldReplicas2.get());
// now the collection should be at RF == 1, with one additional PULL replica
- CloudTestUtils.waitForState(cloudManager, COLL1, 20, TimeUnit.SECONDS,
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 2));
+ }
+
+ @Test
+ public void testDeleteNode() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String COLL1 = "deleteNode_collection";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
+ "conf", 1, 2);
+
+ create.process(solrClient);
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 2));
+
+ // add a couple of spare replicas above RF. Use different types to verify that only
+ // searchable replicas are considered
+ // these additional replicas will be placed on other nodes in the cluster
+ solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.NRT));
+ solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.TLOG));
+ solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.PULL));
+
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 5));
+
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'search_rate_trigger3'," +
+ "'event' : 'searchRate'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : false," +
+ "'collections' : '" + COLL1 + "'," +
+ "'aboveRate' : 1.0," +
+ "'belowRate' : 0.1," +
+ // allow deleting all spare replicas
+ "'minReplicas' : 1," +
+ // allow requesting all deletions in one event
+ "'maxOps' : 10," +
+ // delete underutilised nodes
+ "'belowNodeOp' : 'DELETENODE'," +
+ "'actions' : [" +
+ "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+ "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
+ "]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'started'," +
+ "'trigger' : 'search_rate_trigger3'," +
+ "'stage' : ['STARTED']," +
+ "'class' : '" + StartedProcessingListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'srt'," +
+ "'trigger' : 'search_rate_trigger3'," +
+ "'stage' : ['FAILED','SUCCEEDED']," +
+ "'afterAction': ['compute', 'execute']," +
+ "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'finished'," +
+ "'trigger' : 'search_rate_trigger3'," +
+ "'stage' : ['SUCCEEDED']," +
+ "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
+ // enable the trigger
+ String resumeTriggerCommand = "{" +
+ "'resume-trigger' : {" +
+ "'name' : 'search_rate_trigger3'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
+ boolean await = started.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ await = finished.await(90, TimeUnit.SECONDS);
+ assertTrue("The trigger did not finish processing", await);
+
+ // suspend the trigger
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : 'search_rate_trigger3'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(5000);
+
+ List<CapturedEvent> events = listenerEvents.get("srt");
+ assertEquals(events.toString(), 3, events.size());
+
+ CapturedEvent ev = events.get(0);
+ assertEquals(ev.toString(), "compute", ev.actionName);
+ List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
+ assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
+ // 3 DELETEREPLICA, 3 DELETENODE
+ assertEquals(ops.toString(), 6, ops.size());
+ AtomicInteger replicas = new AtomicInteger();
+ AtomicInteger nodes = new AtomicInteger();
+ ops.forEach(op -> {
+ if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) {
+ replicas.incrementAndGet();
+ } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETENODE)) {
+ nodes.incrementAndGet();
+ } else {
+ fail("unexpected op: " + op);
+ }
+ });
+ assertEquals(ops.toString(), 3, replicas.get());
+ assertEquals(ops.toString(), 3, nodes.get());
+ // check status
+ ev = events.get(1);
+ assertEquals(ev.toString(), "execute", ev.actionName);
+ List<NamedList<Object>> responses = (List<NamedList<Object>>)ev.context.get("properties.responses");
+ assertNotNull(ev.toString(), responses);
+ assertEquals(responses.toString(), 6, responses.size());
+ replicas.set(0);
+ nodes.set(0);
+ responses.forEach(m -> {
+ if (m.get("success") != null) {
+ replicas.incrementAndGet();
+ } else if (m.get("status") != null) {
+ NamedList<Object> status = (NamedList<Object>)m.get("status");
+ if ("completed".equals(status.get("state"))) {
+ nodes.incrementAndGet();
+ } else {
+ fail("unexpected DELETENODE status: " + m);
+ }
+ } else {
+ fail("unexpected status: " + m);
+ }
+ });
+
+ // we are left with one searchable replica and one PULL replica
+ CloudTestUtils.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
CloudTestUtils.clusterShape(1, 2));
}
@@ -489,6 +694,14 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
}
}
+ public static class StartedProcessingListener extends TriggerListenerBase {
+
+ @Override
+ public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception {
+ started.countDown();
+ }
+ }
+
public static class FinishedProcessingListener extends TriggerListenerBase {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/516aad5a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index af3ef98..75c5d76 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -230,20 +230,26 @@ the value is set to 1. Note also that shard leaders are never deleted.
`aboveOp`:: collection action to request when the upper threshold for a shard or replica is
exceeded. Default action is `ADDREPLICA` and the trigger will request 1 to `maxOps` operations
-per shard per event, depending how much the rate is exceeded.
+per shard per event, depending how much the rate is exceeded. This property can be set to 'NONE'
+to effectively disable the action (but still report it to the listeners).
`aboveNodeOp`:: collection action to request when the upper threshold for a node is exceeded.
Default action is `MOVEREPLICA`, and the trigger will request 1 replica operation per hot node per event.
+If both `aboveOp` and `aboveNodeOp` operations are requested then `aboveNodeOp` operations are
+always requested first. This property can be set to 'NONE' to effectively disable the action (but still report it to the listeners).
`belowOp`:: collection action to request when the lower threshold for a shard or replica is
exceeded. Default action is `DELETEREPLICA`, and the trigger will request at most `maxOps` replicas
-to be deleted from eligible cold shards.
+to be deleted from eligible cold shards. This property can be set to 'NONE'
+to effectively disable the action (but still report it to the listeners).
`belowNodeOp`:: action to request when the lower threshold for a node is exceeded.
Default action is null (not set) and the condition is ignored, because in many cases the
trigger will monitor only some selected resources (non-pull replica types from selected
collections / shards) so setting this by default to eg. `DELETENODE` could interfere with
these non-monitored resources. The trigger will request 1 operation per cold node per event.
+If both `belowOp` and `belowNodeOp` operations are requested then `belowOp` operations are
+always requested first.
.Example:
A search rate trigger that monitors collection "test" and adds new replicas if 5-minute