You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/04/17 12:06:38 UTC
lucene-solr:jira/solr-11833: SOLR-11833: Better support for
DELETENODE. Ref Guide documentation.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11833 2e4ac772e -> f4c314012
SOLR-11833: Better support for DELETENODE. Ref Guide documentation.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f4c31401
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f4c31401
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f4c31401
Branch: refs/heads/jira/solr-11833
Commit: f4c314012df1fdfb49d9e547987f17d63b6cc786
Parents: 2e4ac77
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Apr 17 14:06:08 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Apr 17 14:06:08 2018 +0200
----------------------------------------------------------------------
.../cloud/autoscaling/SearchRateTrigger.java | 16 ++--
.../SearchRateTriggerIntegrationTest.java | 1 +
.../src/solrcloud-autoscaling-triggers.adoc | 85 +++++++++++++++-----
.../cloud/autoscaling/DeleteNodeSuggester.java | 46 +++++++++++
.../client/solrj/cloud/autoscaling/Policy.java | 1 +
.../solrj/request/CollectionAdminRequest.java | 7 ++
6 files changed, 128 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f4c31401/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
index 6c9460b..7722b00 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
@@ -139,6 +139,9 @@ public class SearchRateTrigger extends TriggerBase {
if (o != null) {
try {
minReplicas = Integer.parseInt(o.toString());
+ if (minReplicas < 1) {
+ throw new Exception("must be at least 1, or not set to use 'replicationFactor'");
+ }
} catch (Exception e) {
throw new TriggerValidationException(name, MIN_REPLICAS_PROP, "invalid value '" + o + "': " + e.toString());
}
@@ -179,14 +182,13 @@ public class SearchRateTrigger extends TriggerBase {
if (belowOp == null) {
throw new TriggerValidationException(getName(), BELOW_OP_PROP, "unrecognized value: '" + belowOpStr + "'");
}
- Object aboveNodeObj = properties.get(ABOVE_NODE_OP_PROP);
+ Object aboveNodeObj = properties.getOrDefault(ABOVE_NODE_OP_PROP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
+ // do NOT set the default to DELETENODE
Object belowNodeObj = properties.get(BELOW_NODE_OP_PROP);
- if (aboveNodeObj != null) {
- try {
- aboveNodeOp = CollectionParams.CollectionAction.get(String.valueOf(aboveNodeObj));
- } catch (Exception e) {
- throw new TriggerValidationException(getName(), ABOVE_NODE_OP_PROP, "unrecognized value: '" + aboveNodeObj + "'");
- }
+ try {
+ aboveNodeOp = CollectionParams.CollectionAction.get(String.valueOf(aboveNodeObj));
+ } catch (Exception e) {
+ throw new TriggerValidationException(getName(), ABOVE_NODE_OP_PROP, "unrecognized value: '" + aboveNodeObj + "'");
}
if (belowNodeObj != null) {
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f4c31401/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
index f892382..365e6b5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
@@ -284,6 +284,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
"'collections' : '" + COLL1 + "'," +
"'aboveRate' : 1.0," +
"'belowRate' : 0.1," +
+ // do nothing but generate an op
"'belowNodeOp' : 'none'," +
"'actions' : [" +
"{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f4c31401/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index 432a5e9..af3ef98 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -184,43 +184,86 @@ operation `NONE` (which still can be monitored and acted upon by an appropriate
== Search Rate Trigger
-The search rate trigger can be used for monitoring 1-minute average search rates in a selected
-collection, and request that either replicas be moved to different nodes or new replicas be added
-to reduce the per-replica search rate for a collection or shard with search rate hot spots.
-(Future versions of Solr will also be able to automatically remove some replicas
-when search rate falls below the configured lower threshold).
+The search rate trigger can be used for monitoring search rates in a selected
+collection (1-min average rate by default), and request that either replicas be moved from
+"hot nodes" to different nodes, or new replicas be added to "hot shards" to reduce the
+per-replica search rate for a collection or shard with hot spots.
+
+Similarly, if the search rate falls below a threshold then the trigger may request that some
+replicas are deleted from "cold" shards. It can also optionally issue node-level action requests
+when a cumulative node-level rate falls below a threshold.
+
+Note: this trigger calculates node-level cumulative rates using per-replica rates reported by
+replicas that are part of monitored collections / shards. This means that it may report
+some nodes as "cold" (underutilized) because it ignores other, perhaps more active, replicas
+belonging to other collections. Also, nodes that don't host any of the monitored replicas or
+those that are explicitly excluded by `node` config property won't be reported at all.
+
This trigger support the following configuration:
-`collection`:: (string, optional) collection name to monitor, or any collection if empty.
+`collections`:: (string, optional) comma-separated list of collection names to monitor, or any collection if empty / not set.
-`shard`:: (string, optional) shard name within the collection (requires `collection` to be set), or any shard if empty.
+`shard`:: (string, optional) shard name within the collection (requires `collections` to be set to exactly one name), or any shard if empty.
`node`:: (string, optional) node name to monitor, or any if empty.
-`handler`:: (string, optional) handler name whose request rate represents the search rate
-(default is `/select`). This name is used for creating the full metric key, in
-this case `solr.core.<coreName>:QUERY./select.requestTimes:1minRate`.
+`metric`:: (string, optional) metric name that represents the search rate
+(default is `QUERY./select.requestTimes:1minRate`). This name has to identify a single numeric
+metric value, and it may use the colon syntax for selecting one property of a complex metric.
+
+`maxOps`:: (integer, optional) maximum number of add replica / delete replica operations
+requested in a single autoscaling event. The default value is 3 and it helps to smooth out
+the changes to the number of replicas during periods of large search rate fluctuations.
+
+`minReplicas`:: (integer, optional) minimum acceptable number of searchable replicas (ie. replicas other
+than `PULL` type). The trigger will not generate any DELETEREPLICA requests when the number of
+searchable replicas in a shard reaches this threshold. When this value is not set (the default)
+the `replicationFactor` property of the collection is used, and if that property is not set then
+the value is set to 1. Note also that shard leaders are never deleted.
+
+`aboveRate`:: (float) the upper bound for the request rate metric value. At least one of
+`aboveRate` or `belowRate` must be set.
-`rate`:: (double, required) the upper bound for the request rate metric value.
+`belowRate`:: (float) the lower bound for the request rate metric value. At least one of
+`aboveRate` or `belowRate` must be set.
-If a rate is exceeded for a node (but not for individual replicas placed on this node) then
-the action requested by this event is to move one replica (with the highest rate) to another
-node. If a rate is exceeded for a collection or shard then the action requested is to add some
-replicas - currently at least 1 and at most 3, depending on how much the rate is exceeded, proportional to
-the threshold rate and the current request rate.
+`aboveOp`:: collection action to request when the upper threshold for a shard or replica is
+exceeded. Default action is `ADDREPLICA` and the trigger will request 1 to `maxOps` operations
+per shard per event, depending how much the rate is exceeded.
-.Example: a search rate trigger that monitors collection "test" and adds new replicas if 1-minute average request rate of "/select" handler exceeds 100 requests/sec:
+`aboveNodeOp`:: collection action to request when the upper threshold for a node is exceeded.
+Default action is `MOVEREPLICA`, and the trigger will request 1 replica operation per hot node per event.
+
+`belowOp`:: collection action to request when the lower threshold for a shard or replica is
+exceeded. Default action is `DELETEREPLICA`, and the trigger will request at most `maxOps` replicas
+to be deleted from eligible cold shards.
+
+`belowNodeOp`:: action to request when the lower threshold for a node is exceeded.
+Default action is null (not set) and the condition is ignored, because in many cases the
+trigger will monitor only some selected resources (non-pull replica types from selected
+collections / shards) so setting this by default to eg. `DELETENODE` could interfere with
+these non-monitored resources. The trigger will request 1 operation per cold node per event.
+
+.Example:
+A search rate trigger that monitors collection "test" and adds new replicas if 5-minute
+average request rate of "/select" handler exceeds 100 requests/sec, and the condition persists
+for over 20 minutes. If the rate falls below 0.01 and persists for 20 min the trigger will
+request not only replica deletions (leaving at most 1 replica per shard) but also it may
+request node deletion.
[source,json]
----
{
"set-trigger": {
"name" : "search_rate_trigger",
"event" : "searchRate",
- "collection" : "test",
- "handler" : "/select",
- "rate" : 100.0,
- "waitFor" : "1m",
+ "collections" : "test",
+ "metric" : "QUERY./select.requestTimes:5minRate",
+ "aboveRate" : 100.0,
+ "belowRate" : 0.01,
+ "belowNodeOp" : "DELETENODE",
+ "minReplicas" : 1,
+ "waitFor" : "20m",
"enabled" : true,
"actions" : [
{
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f4c31401/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteNodeSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteNodeSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteNodeSuggester.java
new file mode 100644
index 0000000..cfff49e
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteNodeSuggester.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.Set;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.params.CollectionParams;
+
+/**
+ * This suggester produces a DELETENODE request using provided {@link org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint#SRC_NODE}.
+ */
+class DeleteNodeSuggester extends Suggester {
+
+ @Override
+ public CollectionParams.CollectionAction getAction() {
+ return CollectionParams.CollectionAction.DELETENODE;
+ }
+
+ @Override
+ SolrRequest init() {
+ Set<String> srcNodes = (Set<String>) hints.get(Hint.SRC_NODE);
+ if (srcNodes.isEmpty()) {
+ throw new RuntimeException("delete-node requires 'src_node' hint");
+ }
+ if (srcNodes.size() > 1) {
+ throw new RuntimeException("delete-node requires exactly one 'src_node' hint");
+ }
+ return CollectionAdminRequest.deleteNode(srcNodes.iterator().next());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f4c31401/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 2f729d9..048050a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -467,6 +467,7 @@ public class Policy implements MapWriter {
static {
ops.put(CollectionAction.ADDREPLICA, () -> new AddReplicaSuggester());
ops.put(CollectionAction.DELETEREPLICA, () -> new DeleteReplicaSuggester());
+ ops.put(CollectionAction.DELETENODE, () -> new DeleteNodeSuggester());
ops.put(CollectionAction.MOVEREPLICA, () -> new MoveReplicaSuggester());
ops.put(CollectionAction.SPLITSHARD, () -> new SplitShardSuggester());
ops.put(CollectionAction.MERGESHARDS, () -> new UnsupportedSuggester(CollectionAction.MERGESHARDS));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f4c31401/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 96fd4e8..4c74a58 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -553,6 +553,13 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
}
+ /**
+ * Returns a SolrRequest to delete a node.
+ */
+ public static DeleteNode deleteNode(String node) {
+ return new DeleteNode(node);
+ }
+
public static class DeleteNode extends AsyncCollectionAdminRequest {
String node;