You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by no...@apache.org on 2023/06/12 11:01:57 UTC
[solr] branch jira/solr16507_9x created (now 376aaf97fc2)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a change to branch jira/solr16507_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
at 376aaf97fc2 SOLR-16507:Remove NodeStateProvider & Snitch ported to 9x
This branch includes the following new commits:
new 376aaf97fc2 SOLR-16507:Remove NodeStateProvider & Snitch ported to 9x
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[solr] 01/01: SOLR-16507:Remove NodeStateProvider & Snitch ported to 9x
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr16507_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 376aaf97fc2bc5a1a6e3f8545de5e7e99900e2f8
Author: Noble Paul <no...@gmail.com>
AuthorDate: Mon Jun 12 21:01:33 2023 +1000
SOLR-16507:Remove NodeStateProvider & Snitch ported to 9x
---
.../solr/cloud/api/collections/SplitShardCmd.java | 16 +-
.../placement/impl/AttributeFetcherImpl.java | 15 +-
.../cluster/placement/impl/NodeMetricImpl.java | 11 +-
.../solr/client/solrj/impl/NodeValueFetcher.java | 170 ++++++++++++++++++
.../solrj/impl/SolrClientNodeStateProvider.java | 182 +++-----------------
.../solr/common/cloud/rule/ImplicitSnitch.java | 191 ---------------------
.../solr/common/cloud/rule/RemoteCallback.java | 23 ---
.../org/apache/solr/common/cloud/rule/Snitch.java | 29 ----
.../solr/common/cloud/rule/SnitchContext.java | 103 -----------
.../solr/common/cloud/rule/package-info.java | 19 --
.../solrj/request/CollectionAdminRequest.java | 2 -
11 files changed, 214 insertions(+), 547 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 23473694b59..cfbc21743e2 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -17,7 +17,6 @@
package org.apache.solr.cloud.api.collections;
-import static org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Variable.CORE_IDX;
import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.RANDOM;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
@@ -46,11 +45,13 @@ import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.VersionedData;
+import org.apache.solr.client.solrj.impl.NodeValueFetcher;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.cloud.DistributedClusterStateUpdater;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker;
import org.apache.solr.cloud.overseer.OverseerAction;
+import org.apache.solr.cluster.placement.impl.ReplicaMetricImpl;
import org.apache.solr.common.LinkedHashMapWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
@@ -65,7 +66,6 @@ import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.params.CommonParams;
@@ -878,10 +878,12 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
Map<String, Object> nodeValues =
nodeStateProvider.getNodeValues(
- parentShardLeader.getNodeName(), Collections.singletonList(ImplicitSnitch.DISK));
+ parentShardLeader.getNodeName(),
+ Collections.singletonList(NodeValueFetcher.Tags.FREEDISK.tagName));
Map<String, Map<String, List<Replica>>> infos =
nodeStateProvider.getReplicaInfo(
- parentShardLeader.getNodeName(), Collections.singletonList(CORE_IDX.metricsAttribute));
+ parentShardLeader.getNodeName(),
+ Collections.singletonList(ReplicaMetricImpl.INDEX_SIZE_GB.getInternalName()));
if (infos.get(collection) == null || infos.get(collection).get(shard) == null) {
log.warn("cannot verify information for parent shard leader");
return;
@@ -891,12 +893,12 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
Double indexSize = null;
for (Replica info : lst) {
if (info.getCoreName().equals(parentShardLeader.getCoreName())) {
- Number size = (Number) info.get(CORE_IDX.metricsAttribute);
+ Number size = (Number) info.get(ReplicaMetricImpl.INDEX_SIZE_GB.getInternalName());
if (size == null) {
log.warn("cannot verify information for parent shard leader");
return;
}
- indexSize = (Double) CORE_IDX.convertVal(size);
+ indexSize = ReplicaMetricImpl.INDEX_SIZE_GB.convert(size);
break;
}
}
@@ -904,7 +906,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
log.warn("missing replica information for parent shard leader");
return;
}
- Number freeSize = (Number) nodeValues.get(ImplicitSnitch.DISK);
+ Number freeSize = (Number) nodeValues.get(NodeValueFetcher.Tags.FREEDISK.tagName);
if (freeSize == null) {
log.warn("missing node disk space information for parent shard leader");
return;
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
index 0beba2366dc..251dd49759d 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
@@ -27,7 +27,7 @@ import java.util.Set;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
+import org.apache.solr.client.solrj.impl.NodeValueFetcher;
import org.apache.solr.cluster.Node;
import org.apache.solr.cluster.SolrCollection;
import org.apache.solr.cluster.placement.AttributeFetcher;
@@ -37,7 +37,6 @@ import org.apache.solr.cluster.placement.NodeMetric;
import org.apache.solr.cluster.placement.ReplicaMetric;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.slf4j.Logger;
@@ -116,7 +115,7 @@ public class AttributeFetcherImpl implements AttributeFetcher {
for (NodeMetric<?> metric : requestedNodeMetricSnitchTags) {
final Map<Node, Object> metricMap = new HashMap<>();
metricSnitchToNodeToValue.put(metric, metricMap);
- String metricSnitch = getMetricSnitchTag(metric);
+ String metricSnitch = getMetricTag(metric);
allSnitchTagsToInsertion.put(
metricSnitch, (node, value) -> metricMap.put(node, metric.convert(value)));
}
@@ -231,23 +230,23 @@ public class AttributeFetcherImpl implements AttributeFetcher {
}
}
- public static String getMetricSnitchTag(NodeMetric<?> metric) {
+ public static String getMetricTag(NodeMetric<?> metric) {
if (metric.getRegistry() != NodeMetric.Registry.UNSPECIFIED) {
// regular registry + metricName
- return SolrClientNodeStateProvider.METRICS_PREFIX
+ return NodeValueFetcher.METRICS_PREFIX
+ SolrMetricManager.getRegistryName(getGroupFromMetricRegistry(metric.getRegistry()))
+ ":"
+ metric.getInternalName();
- } else if (ImplicitSnitch.tags.contains(metric.getInternalName())) {
+ } else if (NodeValueFetcher.tags.contains(metric.getInternalName())) {
// "special" well-known tag
return metric.getInternalName();
} else {
// a fully-qualified metric key
- return SolrClientNodeStateProvider.METRICS_PREFIX + metric.getInternalName();
+ return NodeValueFetcher.METRICS_PREFIX + metric.getInternalName();
}
}
public static String getSystemPropertySnitchTag(String name) {
- return ImplicitSnitch.SYSPROP + name;
+ return NodeValueFetcher.SYSPROP + name;
}
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java
index f5f9cd8dbd6..b76013d1903 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java
@@ -19,8 +19,8 @@ package org.apache.solr.cluster.placement.impl;
import java.util.Objects;
import java.util.function.Function;
+import org.apache.solr.client.solrj.impl.NodeValueFetcher;
import org.apache.solr.cluster.placement.NodeMetric;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
/**
* Node metric identifier, corresponding to a node-level metric registry and the internal metric
@@ -40,14 +40,17 @@ public class NodeMetricImpl<T> extends MetricImpl<T> implements NodeMetric<T> {
/** Number of all cores. */
public static final NodeMetricImpl<Integer> NUM_CORES =
- new NodeMetricImpl<>(ImplicitSnitch.CORES);
+ new NodeMetricImpl<>(NodeValueFetcher.CORES);
public static final NodeMetricImpl<Double> HEAP_USAGE =
- new NodeMetricImpl<>(ImplicitSnitch.HEAPUSAGE);
+ new NodeMetricImpl<>(NodeValueFetcher.Tags.HEAPUSAGE.tagName);
/** System load average. */
public static final NodeMetricImpl<Double> SYSLOAD_AVG =
- new NodeMetricImpl<>("sysLoadAvg", Registry.SOLR_JVM, "os.systemLoadAverage");
+ new NodeMetricImpl<>(
+ NodeValueFetcher.Tags.SYSLOADAVG.tagName,
+ Registry.SOLR_JVM,
+ NodeValueFetcher.Tags.SYSLOADAVG.prefix);
/** Number of available processors. */
public static final NodeMetricImpl<Integer> AVAILABLE_PROCESSORS =
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
new file mode 100644
index 00000000000..c5832aa1432
--- /dev/null
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.solr.client.solrj.response.SimpleSolrResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+/**
+ * This class is responsible for fetching metrics and other attributes from a given node in Solr
+ * cluster. This is a helper class that is used by {@link SolrClientNodeStateProvider}
+ */
+public class NodeValueFetcher {
+ // well known tags
+ public static final String NODE = "node";
+ public static final String PORT = "port";
+ public static final String HOST = "host";
+ public static final String CORES = "cores";
+ public static final String SYSPROP = "sysprop.";
+ public static final Set<String> tags =
+ Set.of(NODE, PORT, HOST, CORES, Tags.FREEDISK.tagName, Tags.HEAPUSAGE.tagName);
+ public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
+ public static final String METRICS_PREFIX = "metrics:";
+
+ /** Various well known tags that can be fetched from a node */
+ public enum Tags {
+ FREEDISK(
+ "freedisk", "solr.node", "CONTAINER.fs.usableSpace", "solr.node/CONTAINER.fs.usableSpace"),
+ TOTALDISK(
+ "totaldisk", "solr.node", "CONTAINER.fs.totalSpace", "solr.node/CONTAINER.fs.totalSpace"),
+ CORES("cores", "solr.node", "CONTAINER.cores", null) {
+ @Override
+ public Object extractResult(Object root) {
+ NamedList<?> node = (NamedList<?>) Utils.getObjectByPath(root, false, "solr.node");
+ int count = 0;
+ for (String leafCoreMetricName : new String[] {"lazy", "loaded", "unloaded"}) {
+ Number n = (Number) node.get("CONTAINER.cores." + leafCoreMetricName);
+ if (n != null) count += n.intValue();
+ }
+ return count;
+ }
+ },
+ SYSLOADAVG("sysLoadAvg", "solr.jvm", "os.systemLoadAverage", "solr.jvm/os.systemLoadAverage"),
+ HEAPUSAGE("heapUsage", "solr.jvm", "memory.heap.usage", "solr.jvm/memory.heap.usage");
+ // the metrics group
+ public final String group;
+ // the metrics prefix
+ public final String prefix;
+ public final String tagName;
+ // the json path in the response
+ public final String path;
+
+ Tags(String name, String group, String prefix, String path) {
+ this.group = group;
+ this.prefix = prefix;
+ this.tagName = name;
+ this.path = path;
+ }
+
+ public Object extractResult(Object root) {
+ Object v = Utils.getObjectByPath(root, true, path);
+ return v == null ? null : convertVal(v);
+ }
+
+ public Object convertVal(Object val) {
+ if (val instanceof String) {
+ return Double.valueOf((String) val);
+ } else if (val instanceof Number) {
+ Number num = (Number) val;
+ return num.doubleValue();
+
+ } else {
+ throw new IllegalArgumentException("Unknown type : " + val);
+ }
+ }
+ }
+
+ private void getRemoteInfo(
+ String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+ if (!(ctx).isNodeAlive(solrNode)) return;
+ Map<String, Set<Object>> metricsKeyVsTag = new HashMap<>();
+ for (String tag : requestedTags) {
+ if (tag.startsWith(SYSPROP)) {
+ metricsKeyVsTag
+ .computeIfAbsent(
+ "solr.jvm:system.properties:" + tag.substring(SYSPROP.length()),
+ k -> new HashSet<>())
+ .add(tag);
+ } else if (tag.startsWith(METRICS_PREFIX)) {
+ metricsKeyVsTag
+ .computeIfAbsent(tag.substring(METRICS_PREFIX.length()), k -> new HashSet<>())
+ .add(tag);
+ }
+ }
+ if (!metricsKeyVsTag.isEmpty()) {
+ SolrClientNodeStateProvider.fetchReplicaMetrics(solrNode, ctx, metricsKeyVsTag);
+ }
+
+ Set<String> groups = new HashSet<>();
+ List<String> prefixes = new ArrayList<>();
+ for (Tags t : Tags.values()) {
+ if (requestedTags.contains(t.tagName)) {
+ groups.add(t.group);
+ prefixes.add(t.prefix);
+ }
+ }
+ if (groups.isEmpty() || prefixes.isEmpty()) return;
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("group", StrUtils.join(groups, ','));
+ params.add("prefix", StrUtils.join(prefixes, ','));
+
+ try {
+ SimpleSolrResponse rsp = ctx.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);
+ NamedList<?> metrics = (NamedList<?>) rsp.nl.get("metrics");
+ if (metrics != null) {
+ for (Tags t : Tags.values()) {
+ ctx.tags.put(t.tagName, t.extractResult(metrics));
+ }
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error getting remote info", e);
+ }
+ }
+
+ public void getTags(
+ String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+ try {
+ if (requestedTags.contains(NODE)) ctx.tags.put(NODE, solrNode);
+ if (requestedTags.contains(HOST)) {
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ if (hostAndPortMatcher.find()) ctx.tags.put(HOST, hostAndPortMatcher.group(1));
+ }
+ if (requestedTags.contains(PORT)) {
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ if (hostAndPortMatcher.find()) ctx.tags.put(PORT, hostAndPortMatcher.group(2));
+ }
+ getRemoteInfo(solrNode, requestedTags, ctx);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 4d59091db54..1bc2aed24bc 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -17,8 +17,6 @@
package org.apache.solr.client.solrj.impl;
-import static java.util.Collections.emptyMap;
-
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@@ -43,28 +41,22 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
-import org.apache.solr.common.cloud.rule.SnitchContext;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** The <em>real</em> {@link NodeStateProvider}, which communicates with Solr via SolrJ. */
public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter {
- public static final String METRICS_PREFIX = "metrics:";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final CloudLegacySolrClient solrClient;
protected final Map<String, Map<String, Map<String, List<Replica>>>>
nodeVsCollectionVsShardVsReplicaInfo = new HashMap<>();
- private Map<String, Object> snitchSession = new HashMap<>();
@SuppressWarnings({"rawtypes"})
private Map<String, Map> nodeVsTags = new HashMap<>();
@@ -121,10 +113,10 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
}
protected Map<String, Object> fetchTagValues(String node, Collection<String> tags) {
- MetricsFetchingSnitch snitch = new MetricsFetchingSnitch();
- ClientSnitchCtx ctx = new ClientSnitchCtx(null, node, snitchSession, solrClient);
- snitch.getTags(node, new HashSet<>(tags), ctx);
- return ctx.getTags();
+ NodeValueFetcher nodeValueFetcher = new NodeValueFetcher();
+ RemoteCallCtx ctx = new RemoteCallCtx(node, solrClient);
+ nodeValueFetcher.getTags(node, new HashSet<>(tags), ctx);
+ return ctx.tags;
}
public void forEachReplica(String node, Consumer<Replica> consumer) {
@@ -188,13 +180,13 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
Map<String, Set<Object>> collect =
metricsKeyVsTagReplica.entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> Set.of(e.getKey())));
- ClientSnitchCtx ctx = new ClientSnitchCtx(null, null, emptyMap(), solrClient);
+ RemoteCallCtx ctx = new RemoteCallCtx(null, solrClient);
fetchReplicaMetrics(node, ctx, collect);
- return ctx.getTags();
+ return ctx.tags;
}
static void fetchReplicaMetrics(
- String solrNode, ClientSnitchCtx ctx, Map<String, Set<Object>> metricsKeyVsTag) {
+ String solrNode, RemoteCallCtx ctx, Map<String, Set<Object>> metricsKeyVsTag) {
if (!ctx.isNodeAlive(solrNode)) return;
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("key", metricsKeyVsTag.keySet().toArray(new String[0]));
@@ -207,9 +199,9 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
if (tag instanceof Function) {
@SuppressWarnings({"unchecked"})
Pair<String, Object> p = (Pair<String, Object>) ((Function) tag).apply(v);
- ctx.getTags().put(p.first(), p.second());
+ ctx.tags.put(p.first(), p.second());
} else {
- if (v != null) ctx.getTags().put(tag.toString(), v);
+ if (v != null) ctx.tags.put(tag.toString(), v);
}
}
});
@@ -221,110 +213,22 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
@Override
public void close() throws IOException {}
- // uses metrics API to get node information
- static class MetricsFetchingSnitch extends ImplicitSnitch {
- @Override
- protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
- if (!((ClientSnitchCtx) ctx).isNodeAlive(solrNode)) return;
- ClientSnitchCtx snitchContext = (ClientSnitchCtx) ctx;
- Map<String, Set<Object>> metricsKeyVsTag = new HashMap<>();
- for (String tag : requestedTags) {
- if (tag.startsWith(SYSPROP)) {
- metricsKeyVsTag
- .computeIfAbsent(
- "solr.jvm:system.properties:" + tag.substring(SYSPROP.length()),
- k -> new HashSet<>())
- .add(tag);
- } else if (tag.startsWith(METRICS_PREFIX)) {
- metricsKeyVsTag
- .computeIfAbsent(tag.substring(METRICS_PREFIX.length()), k -> new HashSet<>())
- .add(tag);
- }
- }
- if (!metricsKeyVsTag.isEmpty()) {
- fetchReplicaMetrics(solrNode, snitchContext, metricsKeyVsTag);
- }
-
- Set<String> groups = new HashSet<>();
- List<String> prefixes = new ArrayList<>();
- if (requestedTags.contains(DISK)) {
- groups.add("solr.node");
- prefixes.add("CONTAINER.fs.usableSpace");
- }
- if (requestedTags.contains(Variable.TOTALDISK.tagName)) {
- groups.add("solr.node");
- prefixes.add("CONTAINER.fs.totalSpace");
- }
- if (requestedTags.contains(CORES)) {
- groups.add("solr.node");
- prefixes.add("CONTAINER.cores");
- }
- if (requestedTags.contains(SYSLOADAVG)) {
- groups.add("solr.jvm");
- prefixes.add("os.systemLoadAverage");
- }
- if (requestedTags.contains(HEAPUSAGE)) {
- groups.add("solr.jvm");
- prefixes.add("memory.heap.usage");
- }
- if (groups.isEmpty() || prefixes.isEmpty()) return;
-
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add("group", StrUtils.join(groups, ','));
- params.add("prefix", StrUtils.join(prefixes, ','));
-
- try {
- SimpleSolrResponse rsp =
- snitchContext.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);
- NamedList<?> metrics = (NamedList<?>) rsp.nl.get("metrics");
- if (metrics != null) {
- // metrics enabled
- if (requestedTags.contains(Variable.FREEDISK.tagName)) {
- Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.usableSpace");
- if (n != null)
- ctx.getTags().put(Variable.FREEDISK.tagName, Variable.FREEDISK.convertVal(n));
- }
- if (requestedTags.contains(Variable.TOTALDISK.tagName)) {
- Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.totalSpace");
- if (n != null)
- ctx.getTags().put(Variable.TOTALDISK.tagName, Variable.TOTALDISK.convertVal(n));
- }
- if (requestedTags.contains(CORES)) {
- NamedList<?> node = (NamedList<?>) metrics.get("solr.node");
- int count = 0;
- for (String leafCoreMetricName : new String[] {"lazy", "loaded", "unloaded"}) {
- Number n = (Number) node.get("CONTAINER.cores." + leafCoreMetricName);
- if (n != null) count += n.intValue();
- }
- ctx.getTags().put(CORES, count);
- }
- if (requestedTags.contains(SYSLOADAVG)) {
- Number n =
- (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/os.systemLoadAverage");
- if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue());
- }
- if (requestedTags.contains(HEAPUSAGE)) {
- Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/memory.heap.usage");
- if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
- }
- }
- } catch (Exception e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error getting remote info", e);
- }
- }
- }
-
@Override
public String toString() {
return Utils.toJSONString(this);
}
- static class ClientSnitchCtx extends SnitchContext {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ /**
+ * Mostly an info object that stores all the values for a given session to fetch various values
+ * from a node
+ */
+ static class RemoteCallCtx {
ZkClientClusterStateProvider zkClientClusterStateProvider;
CloudLegacySolrClient solrClient;
+ public final Map<String, Object> tags = new HashMap<>();
+ private String node;
+ public Map<String, Object> session;
public boolean isNodeAlive(String node) {
if (zkClientClusterStateProvider != null) {
@@ -333,35 +237,13 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return true;
}
- public ClientSnitchCtx(
- SnitchInfo perSnitch,
- String node,
- Map<String, Object> session,
- CloudLegacySolrClient solrClient) {
- super(perSnitch, node, session);
+ public RemoteCallCtx(String node, CloudLegacySolrClient solrClient) {
+ this.node = node;
this.solrClient = solrClient;
this.zkClientClusterStateProvider =
(ZkClientClusterStateProvider) solrClient.getClusterStateProvider();
}
- @Override
- @SuppressWarnings({"unchecked"})
- public Map<?, ?> getZkJson(String path) throws KeeperException, InterruptedException {
- try {
- byte[] bytes =
- zkClientClusterStateProvider
- .getZkStateReader()
- .getZkClient()
- .getData(path, null, null, true);
- if (bytes != null && bytes.length > 0) {
- return (Map<String, Object>) Utils.fromJSON(bytes);
- }
- } catch (KeeperException.NoNodeException e) {
- return emptyMap();
- }
- return emptyMap();
- }
-
/**
* Will attempt to call {@link #invoke(String, String, SolrParams)} up to five times, retrying
* on any IO Exceptions
@@ -418,31 +300,9 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return request.response;
}
}
- }
-
- public enum Variable {
- FREEDISK("freedisk", null, Double.class),
- TOTALDISK("totaldisk", null, Double.class),
- CORE_IDX("INDEX.sizeInGB", "INDEX.sizeInBytes", Double.class);
- public final String tagName, metricsAttribute;
- public final Class<?> type;
-
- Variable(String tagName, String metricsAttribute, Class<?> type) {
- this.tagName = tagName;
- this.metricsAttribute = metricsAttribute;
- this.type = type;
- }
- public Object convertVal(Object val) {
- if (val instanceof String) {
- return Double.valueOf((String) val);
- } else if (val instanceof Number) {
- Number num = (Number) val;
- return num.doubleValue();
-
- } else {
- throw new IllegalArgumentException("Unknown type : " + val);
- }
+ public String getNode() {
+ return node;
}
}
}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
deleted file mode 100644
index 8f84f34273a..00000000000
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.cloud.rule;
-
-import java.lang.invoke.MethodHandles;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-// This is the client-side component of the snitch
-public class ImplicitSnitch extends Snitch {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
-
- // well known tags
- public static final String NODE = "node";
- public static final String PORT = "port";
- public static final String HOST = "host";
- public static final String CORES = "cores";
- public static final String DISK = "freedisk";
- public static final String ROLE = "role";
- public static final String NODEROLE = "nodeRole";
- public static final String SYSPROP = "sysprop.";
- public static final String SYSLOADAVG = "sysLoadAvg";
- public static final String HEAPUSAGE = "heapUsage";
- public static final List<String> IP_SNITCHES =
- Collections.unmodifiableList(Arrays.asList("ip_1", "ip_2", "ip_3", "ip_4"));
- public static final Set<String> tags =
- Set.of(NODE, PORT, HOST, CORES, DISK, ROLE, HEAPUSAGE, "ip_1", "ip_2", "ip_3", "ip_4");
-
- @Override
- public void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
- try {
- if (requestedTags.contains(NODE)) ctx.getTags().put(NODE, solrNode);
- if (requestedTags.contains(HOST)) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
- if (hostAndPortMatcher.find()) ctx.getTags().put(HOST, hostAndPortMatcher.group(1));
- }
- if (requestedTags.contains(PORT)) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
- if (hostAndPortMatcher.find()) ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
- }
- if (requestedTags.contains(ROLE)) fillRole(solrNode, ctx, ROLE);
- if (requestedTags.contains(NODEROLE))
- fillRole(solrNode, ctx, NODEROLE); // for new policy framework
-
- addIpTags(solrNode, requestedTags, ctx);
-
- getRemoteInfo(solrNode, requestedTags, ctx);
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
-
- protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
- HashMap<String, Object> params = new HashMap<>();
- if (requestedTags.contains(CORES)) params.put(CORES, "1");
- if (requestedTags.contains(DISK)) params.put(DISK, "1");
- for (String tag : requestedTags) {
- if (tag.startsWith(SYSPROP)) params.put(tag, tag.substring(SYSPROP.length()));
- }
-
- if (params.size() > 0) {
- Map<String, Object> vals = ctx.getNodeValues(solrNode, params.keySet());
- for (Map.Entry<String, Object> e : vals.entrySet()) {
- if (e.getValue() != null) params.put(e.getKey(), e.getValue());
- }
- }
- ctx.getTags().putAll(params);
- }
-
- private void fillRole(String solrNode, SnitchContext ctx, String key)
- throws KeeperException, InterruptedException {
- Map<?, ?> roles =
- (Map<?, ?>) ctx.retrieve(ZkStateReader.ROLES); // we don't want to hit the ZK for each node
- // so cache and reuse
- try {
- if (roles == null) roles = ctx.getZkJson(ZkStateReader.ROLES);
- cacheRoles(solrNode, ctx, key, roles);
- } catch (KeeperException.NoNodeException e) {
- cacheRoles(solrNode, ctx, key, Collections.emptyMap());
- }
- }
-
- private void cacheRoles(String solrNode, SnitchContext ctx, String key, Map<?, ?> roles) {
- ctx.store(ZkStateReader.ROLES, roles);
- if (roles != null) {
- for (Map.Entry<?, ?> e : roles.entrySet()) {
- if (e.getValue() instanceof List) {
- if (((List<?>) e.getValue()).contains(solrNode)) {
- ctx.getTags().put(key, e.getKey());
- break;
- }
- }
- }
- }
- }
-
- private static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
-
- @Override
- public boolean isKnownTag(String tag) {
- return tags.contains(tag) || tag.startsWith(SYSPROP);
- }
-
- private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
-
- List<String> requestedHostTags = new ArrayList<>();
- for (String tag : requestedTags) {
- if (IP_SNITCHES.contains(tag)) {
- requestedHostTags.add(tag);
- }
- }
-
- if (requestedHostTags.isEmpty()) {
- return;
- }
-
- String[] ipFragments = getIpFragments(solrNode);
-
- if (ipFragments == null) {
- return;
- }
-
- int ipSnitchCount = IP_SNITCHES.size();
- for (int i = 0; i < ipSnitchCount; i++) {
- String currentTagValue = ipFragments[i];
- String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
-
- if (requestedHostTags.contains(currentTagKey)) {
- context.getTags().put(currentTagKey, currentTagValue);
- }
- }
- }
-
- private String[] getIpFragments(String solrNode) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
- if (hostAndPortMatcher.find()) {
- String host = hostAndPortMatcher.group(1);
- if (host != null) {
- String ip = getHostIp(host);
- if (ip != null) {
- return ip.split(HOST_FRAG_SEPARATOR_REGEX); // IPv6 support will be provided by SOLR-8523
- }
- }
- }
-
- log.warn(
- "Failed to match host IP address from node URL [{}] using regex [{}]",
- solrNode,
- hostAndPortPattern.pattern());
- return null;
- }
-
- public String getHostIp(String host) {
- try {
- InetAddress address = InetAddress.getByName(host);
- return address.getHostAddress();
- } catch (Exception e) {
- log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
- return null;
- }
- }
-}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/RemoteCallback.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/RemoteCallback.java
deleted file mode 100644
index a5a4e98d70f..00000000000
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/RemoteCallback.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.cloud.rule;
-
-import java.util.Map;
-
-public interface RemoteCallback {
- void remoteCallback(SnitchContext ctx, Map<String, Object> returnedVal);
-}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/Snitch.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/Snitch.java
deleted file mode 100644
index 916a639f647..00000000000
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/Snitch.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.cloud.rule;
-
-import java.util.Collections;
-import java.util.Set;
-
-public abstract class Snitch {
- public static final Set<Class<?>> WELL_KNOWN_SNITCHES =
- Collections.singleton(ImplicitSnitch.class);
-
- public abstract void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx);
-
- public abstract boolean isKnownTag(String tag);
-}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java
deleted file mode 100644
index e63660feb5f..00000000000
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.cloud.rule;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This is the context provided to the snitches to interact with the system. This is a
- * per-node-per-snitch instance.
- */
-public abstract class SnitchContext implements RemoteCallback {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final Map<String, Object> tags = new HashMap<>();
- private String node;
- private Map<String, Object> session;
- public final SnitchInfo snitchInfo;
- public Exception exception;
-
- public SnitchContext(SnitchInfo perSnitch, String node, Map<String, Object> session) {
- this.snitchInfo = perSnitch;
- this.node = node;
- this.session = session;
- }
-
- public Map<String, Object> getTags() {
- return tags;
- }
-
- public void store(String s, Object val) {
- if (session != null) session.put(s, val);
- }
-
- public Object retrieve(String s) {
- return session != null ? session.get(s) : null;
- }
-
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return Collections.emptyMap();
- }
-
- public abstract Map<?, ?> getZkJson(String path) throws KeeperException, InterruptedException;
-
- public String getNode() {
- return node;
- }
-
- /**
- * make a call to solrnode/admin/cores with the given params and give a callback. This is designed
- * to be asynchronous because the system would want to batch the calls made to any given node
- *
- * @param node The node for which this call is made
- * @param params The params to be passed to the Snitch counterpart
- * @param klas The name of the class to be invoked in the remote node
- * @param callback The callback to be called when the response is obtained from remote node. If
- * this is passed as null the entire response map will be added as tags
- */
- @Deprecated
- public void invokeRemote(
- String node, ModifiableSolrParams params, String klas, RemoteCallback callback) {}
- ;
-
- @Override
- public void remoteCallback(SnitchContext ctx, Map<String, Object> returnedVal) {
- tags.putAll(returnedVal);
- }
-
- public String getErrMsg() {
- return exception == null ? null : exception.getMessage();
- }
-
- public abstract static class SnitchInfo {
- private final Map<String, Object> conf;
-
- protected SnitchInfo(Map<String, Object> conf) {
- this.conf = conf;
- }
-
- public abstract Set<String> getTagNames();
- }
-}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/package-info.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/package-info.java
deleted file mode 100644
index ba8b61c5d2f..00000000000
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Classes for managing Replica placement strategy when operating in SolrCloud mode. */
-package org.apache.solr.common.cloud.rule;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 0b2357dff0c..83dbd1fae8b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -1343,8 +1343,6 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
- // TODO support rule, snitch
-
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();