You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2018/01/09 11:59:13 UTC
lucene-solr:branch_7x: SOLR-11062: new tag "diskType" in autoscaling
policy
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 8dd24a141 -> bf8de4636
SOLR-11062: new tag "diskType" in autoscaling policy
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bf8de463
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bf8de463
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bf8de463
Branch: refs/heads/branch_7x
Commit: bf8de4636ce8b6f94559cd75d71a8eef4f6352dc
Parents: 8dd24a1
Author: Noble Paul <no...@apache.org>
Authored: Tue Jan 9 22:58:10 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jan 9 22:58:56 2018 +1100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../solr/cloud/autoscaling/TestPolicyCloud.java | 8 ++++--
.../solrj/cloud/autoscaling/Suggestion.java | 11 +++++++-
.../solrj/impl/SolrClientNodeStateProvider.java | 27 +++++++++++++++++---
.../solr/common/cloud/rule/ImplicitSnitch.java | 1 +
5 files changed, 43 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bf8de463/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3ddb45d..8970871 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -60,6 +60,8 @@ New Features
rules that sets the time interval for each collection. An internal Overseer command "ROUTEDALIAS_CREATECOLL"
was created to facilitate this. (David Smiley)
+* SOLR-11062: new tag "diskType" in autoscaling policy (noble)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bf8de463/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
index f53dde1..9637a32 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
@@ -20,11 +20,14 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
+import com.google.common.collect.ImmutableSet;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrRequest;
@@ -227,8 +230,9 @@ public class TestPolicyCloud extends SolrCloudTestCase {
for (String tag : tags) {
assertNotNull( "missing : "+ tag , val.get(tag));
}
-
-
+ val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), Collections.singleton("diskType"));
+ Set<String> diskTypes = ImmutableSet.of("rotational", "ssd");
+ assertTrue(diskTypes.contains(val.get("diskType")));
}
public void testCreateCollectionAddShardWithReplicaTypeUsingPolicy() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bf8de463/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 070869a..51b5046 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -18,9 +18,11 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -34,6 +36,7 @@ import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
+import static java.util.Collections.unmodifiableSet;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -224,7 +227,13 @@ public class Suggestion {
public void getSuggestions(SuggestionCtx ctx) {
perNodeSuggestions(ctx);
}
- },;
+ },
+ DISKTYPE(ImplicitSnitch.DISKTYPE, String.class, unmodifiableSet(new HashSet(Arrays.asList("ssd", "rotational"))), null, null, null) {
+ @Override
+ public void getSuggestions(SuggestionCtx ctx) {
+ perNodeSuggestions(ctx);
+ }
+ };
final Class type;
final Set<String> vals;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bf8de463/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 66f54dd..4bece2c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrRequest;
@@ -142,14 +143,19 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return result;
}
- static void fetchMetrics(String solrNode, ClientSnitchCtx ctx, Map<String, String> metricsKeyVsTag) {
+ static void fetchMetrics(String solrNode, ClientSnitchCtx ctx, Map<String, Object> metricsKeyVsTag) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("key", metricsKeyVsTag.keySet().toArray(new String[metricsKeyVsTag.size()]));
try {
SimpleSolrResponse rsp = ctx.invoke(solrNode, CommonParams.METRICS_PATH, params);
metricsKeyVsTag.forEach((key, tag) -> {
Object v = Utils.getObjectByPath(rsp.nl, true, Arrays.asList("metrics", key));
- if (v != null) ctx.getTags().put(tag, v);
+ if (tag instanceof Function) {
+ Pair<String, Object> p = (Pair<String, Object>) ((Function) tag).apply(v);
+ ctx.getTags().put(p.first(), p.second());
+ } else {
+ if (v != null) ctx.getTags().put(tag.toString(), v);
+ }
});
} catch (Exception e) {
log.warn("could not get tags from node " + solrNode, e);
@@ -166,7 +172,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
@Override
protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
ClientSnitchCtx snitchContext = (ClientSnitchCtx) ctx;
- Map<String, String> metricsKeyVsTag = new HashMap<>();
+ Map<String, Object> metricsKeyVsTag = new HashMap<>();
for (String tag : requestedTags) {
if (tag.startsWith(SYSPROP)) {
metricsKeyVsTag.put("solr.jvm:system.properties:" + tag.substring(SYSPROP.length()), tag);
@@ -174,6 +180,21 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
metricsKeyVsTag.put(tag.substring(METRICS_PREFIX.length()), tag);
}
}
+ if (requestedTags.contains(ImplicitSnitch.DISKTYPE)) {
+ metricsKeyVsTag.put("solr.node:CONTAINER.fs.coreRoot.spins", new Function<Object, Pair<String,Object>>() {
+ @Override
+ public Pair<String, Object> apply(Object o) {
+ if("true".equals(String.valueOf(o))){
+ return new Pair<>(ImplicitSnitch.DISKTYPE, "rotational");
+ }
+ if("false".equals(String.valueOf(o))){
+ return new Pair<>(ImplicitSnitch.DISKTYPE, "ssd");
+ }
+ return new Pair<>(ImplicitSnitch.DISKTYPE,null);
+
+ }
+ });
+ }
if (!metricsKeyVsTag.isEmpty()) {
fetchMetrics(solrNode, snitchContext, metricsKeyVsTag);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bf8de463/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
index e087342..ec55ccb 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
@@ -52,6 +52,7 @@ public class ImplicitSnitch extends Snitch {
public static final String SYSPROP = "sysprop.";
public static final String SYSLOADAVG = "sysLoadAvg";
public static final String HEAPUSAGE = "heapUsage";
+ public static final String DISKTYPE = "diskType";
public static final List<String> IP_SNITCHES = Collections.unmodifiableList(Arrays.asList("ip_1", "ip_2", "ip_3", "ip_4"));
public static final Set<String> tags = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(NODE, PORT, HOST, CORES, DISK, ROLE, "ip_1", "ip_2", "ip_3", "ip_4")));