You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/04/22 11:44:00 UTC
[lucene-solr] branch jira/solr-12845 updated (44b215d -> 8444240)
This is an automated email from the ASF dual-hosted git repository.
ab pushed a change to branch jira/solr-12845
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 44b215d SOLR-12845: Fix test failures. Expose the default policy by saving it on OverseerTriggerThread startup.
new cfa3853 SOLR-12845: Update the RefGuide.
new 8444240 SOLR-12845: Fix tests, a bug in SimScenario that led to dropped per-node values and a non-compliance in SimNodeStateProvider.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../cloud/autoscaling/sim/SimNodeStateProvider.java | 11 ++++++++++-
.../solr/cloud/autoscaling/sim/SimScenario.java | 4 +++-
.../cloud/autoscaling/sim/TestSimLargeCluster.java | 2 ++
.../solr/cloud/autoscaling/sim/TestSimScenario.java | 2 ++
.../src/solrcloud-autoscaling-overview.adoc | 21 +++++++++++++++++++++
.../solr/client/solrj/cloud/autoscaling/Policy.java | 7 +++++--
6 files changed, 43 insertions(+), 4 deletions(-)
[lucene-solr] 01/02: SOLR-12845: Update the RefGuide.
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-12845
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit cfa3853b2e6ac1b83bb42c8b242a9e22d57ed087
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Tue Apr 21 14:01:51 2020 +0200
SOLR-12845: Update the RefGuide.
---
.../src/solrcloud-autoscaling-overview.adoc | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
index b7c438c..8582e63 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
@@ -65,6 +65,27 @@ There are many metrics on which the rule can be based, e.g., system load average
When a node, shard, or collection does not satisfy a policy rule, we call it a *violation*. By default, cluster management operations will fail if there is even one violation. You can allow operations to succeed in the face of a violation by marking the corresponding rule with <<solrcloud-autoscaling-policy-preferences.adoc#rule-strictness,`"strict":false`>>. When you do this, Solr ensures that cluster management operations minimize the number of violations.
+The default cluster policy, if none is specified, is the following:
+
+[source,json]
+----
+[
+ { "replica" : "<2", "shard" : "#EACH", "node" : "#ANY", "strict" : false},
+ { "replica" : "#EQUAL", "node" : "#ANY", "strict" : false},
+ { "cores" : "#EQUAL", "node" : "#ANY", "strict" : false}
+]
+----
+
+These rules mean that:
+
+* Each shard should not have more than one replica on the same node, if possible (strict: false).
+* Each collection's replicas should be equally distributed among nodes.
+* All cores should be equally distributed among nodes.
+
+
+NOTE: You can remove this default policy by specifying an empty rule-set in the autoscaling
+admin request, like this: `{set-cluster-policy : []}`.
+
Solr also supports <<solrcloud-autoscaling-policy-preferences.adoc#collection-specific-policy,collection-specific policies>>, which operate in tandem with the cluster policy.
== Triggers
[lucene-solr] 02/02: SOLR-12845: Fix tests,
a bug in SimScenario that led to dropped per-node values and a
non-compliance in SimNodeStateProvider.
Posted by ab...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch jira/solr-12845
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 8444240d4011309bdbc616617fc299a050116d15
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Apr 22 13:42:46 2020 +0200
SOLR-12845: Fix tests, a bug in SimScenario that led to dropped per-node
values and a non-compliance in SimNodeStateProvider.
---
.../solr/cloud/autoscaling/sim/SimNodeStateProvider.java | 11 ++++++++++-
.../org/apache/solr/cloud/autoscaling/sim/SimScenario.java | 4 +++-
.../solr/cloud/autoscaling/sim/TestSimLargeCluster.java | 2 ++
.../apache/solr/cloud/autoscaling/sim/TestSimScenario.java | 2 ++
.../apache/solr/client/solrj/cloud/autoscaling/Policy.java | 7 +++++--
5 files changed, 22 insertions(+), 4 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
index 3a9d775..9074614 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
@@ -19,6 +19,7 @@ package org.apache.solr.cloud.autoscaling.sim;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -233,6 +234,11 @@ public class SimNodeStateProvider implements NodeStateProvider {
return nodeValues;
}
+ /** Get all values for a selected node. */
+ public Map<String, Object> simGetNodeValues(String node) {
+ return nodeValues.getOrDefault(node, Collections.emptyMap());
+ }
+
private void saveRoles() {
final Map<String, Set<String>> roles = new HashMap<>();
nodeValues.forEach((n, values) -> {
@@ -319,7 +325,10 @@ public class SimNodeStateProvider implements NodeStateProvider {
if (values == null) {
return result;
}
- result.putAll(values.entrySet().stream().filter(e -> tags.contains(e.getKey())).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())));
+ result.putAll(values.entrySet().stream()
+ .filter(e -> tags.contains(e.getKey()))
+ .filter(e -> e.getValue() != null)
+ .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())));
return result;
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimScenario.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimScenario.java
index 40e8716..76bd53f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimScenario.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimScenario.java
@@ -803,7 +803,9 @@ public class SimScenario implements AutoCloseable {
values.put(key, val);
}
for (String node : nodes) {
- scenario.cluster.getSimNodeStateProvider().simSetNodeValues(node, values);
+ Map<String, Object> newValues = new HashMap<>(scenario.cluster.getSimNodeStateProvider().simGetNodeValues(node));
+ newValues.putAll(values);
+ scenario.cluster.getSimNodeStateProvider().simSetNodeValues(node, newValues);
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
index 2efcaeb..85462f2 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
@@ -100,6 +100,8 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
// disable metrics history collection
cluster.disableMetricsHistory();
+ // turn off the default policy to avoid slowdowns due to the costly #EQUAL rules
+ CloudTestUtils.assertAutoScalingRequest(cluster, "{'set-cluster-policy': []}");
// disable .scheduled_maintenance (once it exists)
CloudTestUtils.waitForTriggerToBeScheduled(cluster, ".scheduled_maintenance");
CloudTestUtils.suspendTrigger(cluster, ".scheduled_maintenance");
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimScenario.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimScenario.java
index e08999b..c87fccf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimScenario.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimScenario.java
@@ -119,6 +119,7 @@ public class TestSimScenario extends SimSolrCloudTestCase {
String indexingScenario =
"create_cluster numNodes=100\n" +
+ "load_autoscaling json={'cluster-policy':[]}\n" +
"solr_request /admin/collections?action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=2\n" +
"wait_collection collection=testCollection&shards=2&replicas=2\n" +
"solr_request /admin/autoscaling?httpMethod=POST&stream.body=" +
@@ -142,6 +143,7 @@ public class TestSimScenario extends SimSolrCloudTestCase {
String splitShardScenario =
"create_cluster numNodes=2\n" +
+ "load_autoscaling json={'cluster-policy':[]}\n" +
"solr_request /admin/collections?action=CREATE&name=testCollection&numShards=2&replicationFactor=2&maxShardsPerNode=5\n" +
"wait_collection collection=testCollection&shards=2&replicas=2\n" +
"set_shard_metrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000\n" +
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index a1294c1..3448661 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -18,6 +18,7 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
+import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@@ -331,11 +332,13 @@ public class Policy implements MapWriter {
PolicyHelper.writeNodes(ew, matrixCopy);
ew.put("config", matrix.get(0).session.getPolicy());
});
- log.error("Exception! prefs = {}, recent r1 = {}, r2 = {}, matrix = {}",
+ StringWriter exc = new StringWriter();
+ e.printStackTrace(new PrintWriter(exc));
+ log.error("Exception during matrix sorting! prefs = {}, recent r1 = {}, r2 = {}, matrix = {}, exception={}",
clusterPreferences,
lastComparison[0].node,
lastComparison[1].node,
- Utils.writeJson(m, new StringWriter(), true).toString());
+ Utils.writeJson(m, new StringWriter(), true).toString(), exc.toString());
} catch (IOException e1) {
//
}