You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by no...@apache.org on 2023/05/04 03:36:56 UTC
[solr] branch jira/solr16507_refactor created (now f64f5759fc3)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a change to branch jira/solr16507_refactor
in repository https://gitbox.apache.org/repos/asf/solr.git
at f64f5759fc3 refactored out ImplicitSnitch
This branch includes the following new commits:
new f64f5759fc3 refactored out ImplicitSnitch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[solr] 01/01: refactored out ImplicitSnitch
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr16507_refactor
in repository https://gitbox.apache.org/repos/asf/solr.git
commit f64f5759fc39d1a4f4ecb228c62f12259d45e954
Author: Noble Paul <no...@gmail.com>
AuthorDate: Thu May 4 13:36:38 2023 +1000
refactored out ImplicitSnitch
---
.../placement/impl/AttributeFetcherImpl.java | 6 +-
.../cluster/placement/impl/NodeMetricImpl.java | 7 +-
.../solr/client/solrj/impl/MetricsFetcher.java | 253 +++++++++++++++++++++
.../solrj/impl/SolrClientNodeStateProvider.java | 108 +--------
.../solr/common/cloud/rule/ImplicitSnitch.java | 191 ----------------
.../org/apache/solr/common/cloud/rule/Snitch.java | 29 ---
.../solr/common/cloud/rule/SnitchContext.java | 53 +----
7 files changed, 268 insertions(+), 379 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
index 0beba2366dc..0cbcba70e5a 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
@@ -27,6 +27,7 @@ import java.util.Set;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.impl.MetricsFetcher;
import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
import org.apache.solr.cluster.Node;
import org.apache.solr.cluster.SolrCollection;
@@ -37,7 +38,6 @@ import org.apache.solr.cluster.placement.NodeMetric;
import org.apache.solr.cluster.placement.ReplicaMetric;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.slf4j.Logger;
@@ -238,7 +238,7 @@ public class AttributeFetcherImpl implements AttributeFetcher {
+ SolrMetricManager.getRegistryName(getGroupFromMetricRegistry(metric.getRegistry()))
+ ":"
+ metric.getInternalName();
- } else if (ImplicitSnitch.tags.contains(metric.getInternalName())) {
+ } else if (MetricsFetcher.tags.contains(metric.getInternalName())) {
// "special" well-known tag
return metric.getInternalName();
} else {
@@ -248,6 +248,6 @@ public class AttributeFetcherImpl implements AttributeFetcher {
}
public static String getSystemPropertySnitchTag(String name) {
- return ImplicitSnitch.SYSPROP + name;
+ return MetricsFetcher.SYSPROP + name;
}
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java
index f5f9cd8dbd6..74a5d025e2d 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java
@@ -19,8 +19,9 @@ package org.apache.solr.cluster.placement.impl;
import java.util.Objects;
import java.util.function.Function;
+
+import org.apache.solr.client.solrj.impl.MetricsFetcher;
import org.apache.solr.cluster.placement.NodeMetric;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
/**
* Node metric identifier, corresponding to a node-level metric registry and the internal metric
@@ -40,10 +41,10 @@ public class NodeMetricImpl<T> extends MetricImpl<T> implements NodeMetric<T> {
/** Number of all cores. */
public static final NodeMetricImpl<Integer> NUM_CORES =
- new NodeMetricImpl<>(ImplicitSnitch.CORES);
+ new NodeMetricImpl<>(MetricsFetcher.CORES);
public static final NodeMetricImpl<Double> HEAP_USAGE =
- new NodeMetricImpl<>(ImplicitSnitch.HEAPUSAGE);
+ new NodeMetricImpl<>(MetricsFetcher.HEAPUSAGE);
/** System load average. */
public static final NodeMetricImpl<Double> SYSLOAD_AVG =
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/MetricsFetcher.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/MetricsFetcher.java
new file mode 100644
index 00000000000..9b067444fd9
--- /dev/null
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/MetricsFetcher.java
@@ -0,0 +1,253 @@
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.solr.client.solrj.response.SimpleSolrResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.rule.SnitchContext;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// uses metrics API to get node information
+public class MetricsFetcher {
+ // well known tags
+ public static final String NODE = "node";
+ public static final String PORT = "port";
+ public static final String HOST = "host";
+ public static final String CORES = "cores";
+ public static final String DISK = "freedisk";
+ public static final String ROLE = "role";
+ public static final String NODEROLE = "nodeRole";
+ public static final String SYSPROP = "sysprop.";
+ public static final String SYSLOADAVG = "sysLoadAvg";
+ public static final String HEAPUSAGE = "heapUsage";
+ public static final Set<String> tags =
+ Set.of(NODE, PORT, HOST, CORES, DISK, ROLE, HEAPUSAGE, "ip_1", "ip_2", "ip_3", "ip_4");
+ public static final List<String> IP_SNITCHES =
+ Collections.unmodifiableList(Arrays.asList("ip_1", "ip_2", "ip_3", "ip_4"));
+ public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
+ if (!((SolrClientNodeStateProvider.ClientSnitchCtx) ctx).isNodeAlive(solrNode)) return;
+ SolrClientNodeStateProvider.ClientSnitchCtx snitchContext = (SolrClientNodeStateProvider.ClientSnitchCtx) ctx;
+ Map<String, Set<Object>> metricsKeyVsTag = new HashMap<>();
+ for (String tag : requestedTags) {
+ if (tag.startsWith(SYSPROP)) {
+ metricsKeyVsTag
+ .computeIfAbsent(
+ "solr.jvm:system.properties:" + tag.substring(SYSPROP.length()),
+ k -> new HashSet<>())
+ .add(tag);
+ } else if (tag.startsWith(SolrClientNodeStateProvider.METRICS_PREFIX)) {
+ metricsKeyVsTag
+ .computeIfAbsent(tag.substring(SolrClientNodeStateProvider.METRICS_PREFIX.length()), k -> new HashSet<>())
+ .add(tag);
+ }
+ }
+ if (!metricsKeyVsTag.isEmpty()) {
+ SolrClientNodeStateProvider.fetchReplicaMetrics(solrNode, snitchContext, metricsKeyVsTag);
+ }
+
+ Set<String> groups = new HashSet<>();
+ List<String> prefixes = new ArrayList<>();
+ if (requestedTags.contains(DISK)) {
+ groups.add("solr.node");
+ prefixes.add("CONTAINER.fs.usableSpace");
+ }
+ if (requestedTags.contains(SolrClientNodeStateProvider.Variable.TOTALDISK.tagName)) {
+ groups.add("solr.node");
+ prefixes.add("CONTAINER.fs.totalSpace");
+ }
+ if (requestedTags.contains(CORES)) {
+ groups.add("solr.node");
+ prefixes.add("CONTAINER.cores");
+ }
+ if (requestedTags.contains(SYSLOADAVG)) {
+ groups.add("solr.jvm");
+ prefixes.add("os.systemLoadAverage");
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ groups.add("solr.jvm");
+ prefixes.add("memory.heap.usage");
+ }
+ if (groups.isEmpty() || prefixes.isEmpty()) return;
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("group", StrUtils.join(groups, ','));
+ params.add("prefix", StrUtils.join(prefixes, ','));
+
+ try {
+ SimpleSolrResponse rsp =
+ snitchContext.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);
+ NamedList<?> metrics = (NamedList<?>) rsp.nl.get("metrics");
+ if (metrics != null) {
+ // metrics enabled
+ if (requestedTags.contains(SolrClientNodeStateProvider.Variable.FREEDISK.tagName)) {
+ Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.usableSpace");
+ if (n != null)
+ ctx.getTags().put(SolrClientNodeStateProvider.Variable.FREEDISK.tagName, SolrClientNodeStateProvider.Variable.FREEDISK.convertVal(n));
+ }
+ if (requestedTags.contains(SolrClientNodeStateProvider.Variable.TOTALDISK.tagName)) {
+ Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.totalSpace");
+ if (n != null)
+ ctx.getTags().put(SolrClientNodeStateProvider.Variable.TOTALDISK.tagName, SolrClientNodeStateProvider.Variable.TOTALDISK.convertVal(n));
+ }
+ if (requestedTags.contains(CORES)) {
+ NamedList<?> node = (NamedList<?>) metrics.get("solr.node");
+ int count = 0;
+ for (String leafCoreMetricName : new String[]{"lazy", "loaded", "unloaded"}) {
+ Number n = (Number) node.get("CONTAINER.cores." + leafCoreMetricName);
+ if (n != null) count += n.intValue();
+ }
+ ctx.getTags().put(CORES, count);
+ }
+ if (requestedTags.contains(SYSLOADAVG)) {
+ Number n =
+ (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/os.systemLoadAverage");
+ if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue());
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/memory.heap.usage");
+ if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
+ }
+ }
+ } catch (Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error getting remote info", e);
+ }
+ }
+
+ public void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
+ try {
+ if (requestedTags.contains(NODE))
+ ctx.getTags().put(NODE, solrNode);
+ if (requestedTags.contains(HOST)) {
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ if (hostAndPortMatcher.find())
+ ctx.getTags().put(HOST, hostAndPortMatcher.group(1));
+ }
+ if (requestedTags.contains(PORT)) {
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ if (hostAndPortMatcher.find())
+ ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
+ }
+ if (requestedTags.contains(ROLE))
+ fillRole(solrNode, ctx, ROLE);
+ if (requestedTags.contains(NODEROLE))
+ fillRole(solrNode, ctx, NODEROLE); // for new policy framework
+
+ addIpTags(solrNode, requestedTags, ctx);
+
+ getRemoteInfo(solrNode, requestedTags, ctx);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
+
+ List<String> requestedHostTags = new ArrayList<>();
+ for (String tag : requestedTags) {
+ if (IP_SNITCHES.contains(tag)) {
+ requestedHostTags.add(tag);
+ }
+ }
+
+ if (requestedHostTags.isEmpty()) {
+ return;
+ }
+
+ String[] ipFragments = getIpFragments(solrNode);
+
+ if (ipFragments == null) {
+ return;
+ }
+
+ int ipSnitchCount = IP_SNITCHES.size();
+ for (int i = 0; i < ipSnitchCount; i++) {
+ String currentTagValue = ipFragments[i];
+ String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
+
+ if (requestedHostTags.contains(currentTagKey)) {
+ context.getTags().put(currentTagKey, currentTagValue);
+ }
+ }
+ }
+
+ private String[] getIpFragments(String solrNode) {
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ if (hostAndPortMatcher.find()) {
+ String host = hostAndPortMatcher.group(1);
+ if (host != null) {
+ String ip = getHostIp(host);
+ if (ip != null) {
+ return ip.split(SolrClientNodeStateProvider.HOST_FRAG_SEPARATOR_REGEX); // IPv6 support will be provided by SOLR-8523
+ }
+ }
+ }
+
+ log.warn(
+ "Failed to match host IP address from node URL [{}] using regex [{}]",
+ solrNode,
+ hostAndPortPattern.pattern());
+ return null;
+ }
+
+ public String getHostIp(String host) {
+ try {
+ InetAddress address = InetAddress.getByName(host);
+ return address.getHostAddress();
+ } catch (Exception e) {
+ log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
+ return null;
+ }
+ }
+
+ private void fillRole(String solrNode, SnitchContext ctx, String key)
+ throws KeeperException, InterruptedException {
+ Map<?, ?> roles =
+ (Map<?, ?>) ctx.retrieve(ZkStateReader.ROLES); // we don't want to hit the ZK for each node
+ // so cache and reuse
+ try {
+ if (roles == null) roles = ctx.getZkJson(ZkStateReader.ROLES);
+ cacheRoles(solrNode, ctx, key, roles);
+ } catch (KeeperException.NoNodeException e) {
+ cacheRoles(solrNode, ctx, key, Collections.emptyMap());
+ }
+ }
+
+ private void cacheRoles(String solrNode, SnitchContext ctx, String key, Map<?, ?> roles) {
+ ctx.store(ZkStateReader.ROLES, roles);
+ if (roles != null) {
+ for (Map.Entry<?, ?> e : roles.entrySet()) {
+ if (e.getValue() instanceof List) {
+ if (((List<?>) e.getValue()).contains(solrNode)) {
+ ctx.getTags().put(key, e.getKey());
+ break;
+ }
+ }
+ }
+ }
+ }
+
+
+}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 4d59091db54..57b759bd3eb 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -43,14 +43,12 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.cloud.rule.SnitchContext;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -60,6 +58,7 @@ import org.slf4j.LoggerFactory;
public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter {
public static final String METRICS_PREFIX = "metrics:";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
private final CloudLegacySolrClient solrClient;
protected final Map<String, Map<String, Map<String, List<Replica>>>>
@@ -121,9 +120,9 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
}
protected Map<String, Object> fetchTagValues(String node, Collection<String> tags) {
- MetricsFetchingSnitch snitch = new MetricsFetchingSnitch();
- ClientSnitchCtx ctx = new ClientSnitchCtx(null, node, snitchSession, solrClient);
- snitch.getTags(node, new HashSet<>(tags), ctx);
+ MetricsFetcher metricsFetcher = new MetricsFetcher();
+ ClientSnitchCtx ctx = new ClientSnitchCtx(node, snitchSession, solrClient);
+ metricsFetcher.getTags(node, new HashSet<>(tags), ctx);
return ctx.getTags();
}
@@ -188,7 +187,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
Map<String, Set<Object>> collect =
metricsKeyVsTagReplica.entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> Set.of(e.getKey())));
- ClientSnitchCtx ctx = new ClientSnitchCtx(null, null, emptyMap(), solrClient);
+ ClientSnitchCtx ctx = new ClientSnitchCtx(null, emptyMap(), solrClient);
fetchReplicaMetrics(node, ctx, collect);
return ctx.getTags();
}
@@ -221,100 +220,6 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
@Override
public void close() throws IOException {}
- // uses metrics API to get node information
- static class MetricsFetchingSnitch extends ImplicitSnitch {
- @Override
- protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
- if (!((ClientSnitchCtx) ctx).isNodeAlive(solrNode)) return;
- ClientSnitchCtx snitchContext = (ClientSnitchCtx) ctx;
- Map<String, Set<Object>> metricsKeyVsTag = new HashMap<>();
- for (String tag : requestedTags) {
- if (tag.startsWith(SYSPROP)) {
- metricsKeyVsTag
- .computeIfAbsent(
- "solr.jvm:system.properties:" + tag.substring(SYSPROP.length()),
- k -> new HashSet<>())
- .add(tag);
- } else if (tag.startsWith(METRICS_PREFIX)) {
- metricsKeyVsTag
- .computeIfAbsent(tag.substring(METRICS_PREFIX.length()), k -> new HashSet<>())
- .add(tag);
- }
- }
- if (!metricsKeyVsTag.isEmpty()) {
- fetchReplicaMetrics(solrNode, snitchContext, metricsKeyVsTag);
- }
-
- Set<String> groups = new HashSet<>();
- List<String> prefixes = new ArrayList<>();
- if (requestedTags.contains(DISK)) {
- groups.add("solr.node");
- prefixes.add("CONTAINER.fs.usableSpace");
- }
- if (requestedTags.contains(Variable.TOTALDISK.tagName)) {
- groups.add("solr.node");
- prefixes.add("CONTAINER.fs.totalSpace");
- }
- if (requestedTags.contains(CORES)) {
- groups.add("solr.node");
- prefixes.add("CONTAINER.cores");
- }
- if (requestedTags.contains(SYSLOADAVG)) {
- groups.add("solr.jvm");
- prefixes.add("os.systemLoadAverage");
- }
- if (requestedTags.contains(HEAPUSAGE)) {
- groups.add("solr.jvm");
- prefixes.add("memory.heap.usage");
- }
- if (groups.isEmpty() || prefixes.isEmpty()) return;
-
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add("group", StrUtils.join(groups, ','));
- params.add("prefix", StrUtils.join(prefixes, ','));
-
- try {
- SimpleSolrResponse rsp =
- snitchContext.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);
- NamedList<?> metrics = (NamedList<?>) rsp.nl.get("metrics");
- if (metrics != null) {
- // metrics enabled
- if (requestedTags.contains(Variable.FREEDISK.tagName)) {
- Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.usableSpace");
- if (n != null)
- ctx.getTags().put(Variable.FREEDISK.tagName, Variable.FREEDISK.convertVal(n));
- }
- if (requestedTags.contains(Variable.TOTALDISK.tagName)) {
- Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.totalSpace");
- if (n != null)
- ctx.getTags().put(Variable.TOTALDISK.tagName, Variable.TOTALDISK.convertVal(n));
- }
- if (requestedTags.contains(CORES)) {
- NamedList<?> node = (NamedList<?>) metrics.get("solr.node");
- int count = 0;
- for (String leafCoreMetricName : new String[] {"lazy", "loaded", "unloaded"}) {
- Number n = (Number) node.get("CONTAINER.cores." + leafCoreMetricName);
- if (n != null) count += n.intValue();
- }
- ctx.getTags().put(CORES, count);
- }
- if (requestedTags.contains(SYSLOADAVG)) {
- Number n =
- (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/os.systemLoadAverage");
- if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue());
- }
- if (requestedTags.contains(HEAPUSAGE)) {
- Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/memory.heap.usage");
- if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
- }
- }
- } catch (Exception e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error getting remote info", e);
- }
- }
- }
-
@Override
public String toString() {
return Utils.toJSONString(this);
@@ -334,11 +239,10 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
}
public ClientSnitchCtx(
- SnitchInfo perSnitch,
String node,
Map<String, Object> session,
CloudLegacySolrClient solrClient) {
- super(perSnitch, node, session);
+ super( node, session);
this.solrClient = solrClient;
this.zkClientClusterStateProvider =
(ZkClientClusterStateProvider) solrClient.getClusterStateProvider();
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
deleted file mode 100644
index 8f84f34273a..00000000000
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.cloud.rule;
-
-import java.lang.invoke.MethodHandles;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-// This is the client-side component of the snitch
-public class ImplicitSnitch extends Snitch {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
-
- // well known tags
- public static final String NODE = "node";
- public static final String PORT = "port";
- public static final String HOST = "host";
- public static final String CORES = "cores";
- public static final String DISK = "freedisk";
- public static final String ROLE = "role";
- public static final String NODEROLE = "nodeRole";
- public static final String SYSPROP = "sysprop.";
- public static final String SYSLOADAVG = "sysLoadAvg";
- public static final String HEAPUSAGE = "heapUsage";
- public static final List<String> IP_SNITCHES =
- Collections.unmodifiableList(Arrays.asList("ip_1", "ip_2", "ip_3", "ip_4"));
- public static final Set<String> tags =
- Set.of(NODE, PORT, HOST, CORES, DISK, ROLE, HEAPUSAGE, "ip_1", "ip_2", "ip_3", "ip_4");
-
- @Override
- public void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
- try {
- if (requestedTags.contains(NODE)) ctx.getTags().put(NODE, solrNode);
- if (requestedTags.contains(HOST)) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
- if (hostAndPortMatcher.find()) ctx.getTags().put(HOST, hostAndPortMatcher.group(1));
- }
- if (requestedTags.contains(PORT)) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
- if (hostAndPortMatcher.find()) ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
- }
- if (requestedTags.contains(ROLE)) fillRole(solrNode, ctx, ROLE);
- if (requestedTags.contains(NODEROLE))
- fillRole(solrNode, ctx, NODEROLE); // for new policy framework
-
- addIpTags(solrNode, requestedTags, ctx);
-
- getRemoteInfo(solrNode, requestedTags, ctx);
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
- }
-
- protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
- HashMap<String, Object> params = new HashMap<>();
- if (requestedTags.contains(CORES)) params.put(CORES, "1");
- if (requestedTags.contains(DISK)) params.put(DISK, "1");
- for (String tag : requestedTags) {
- if (tag.startsWith(SYSPROP)) params.put(tag, tag.substring(SYSPROP.length()));
- }
-
- if (params.size() > 0) {
- Map<String, Object> vals = ctx.getNodeValues(solrNode, params.keySet());
- for (Map.Entry<String, Object> e : vals.entrySet()) {
- if (e.getValue() != null) params.put(e.getKey(), e.getValue());
- }
- }
- ctx.getTags().putAll(params);
- }
-
- private void fillRole(String solrNode, SnitchContext ctx, String key)
- throws KeeperException, InterruptedException {
- Map<?, ?> roles =
- (Map<?, ?>) ctx.retrieve(ZkStateReader.ROLES); // we don't want to hit the ZK for each node
- // so cache and reuse
- try {
- if (roles == null) roles = ctx.getZkJson(ZkStateReader.ROLES);
- cacheRoles(solrNode, ctx, key, roles);
- } catch (KeeperException.NoNodeException e) {
- cacheRoles(solrNode, ctx, key, Collections.emptyMap());
- }
- }
-
- private void cacheRoles(String solrNode, SnitchContext ctx, String key, Map<?, ?> roles) {
- ctx.store(ZkStateReader.ROLES, roles);
- if (roles != null) {
- for (Map.Entry<?, ?> e : roles.entrySet()) {
- if (e.getValue() instanceof List) {
- if (((List<?>) e.getValue()).contains(solrNode)) {
- ctx.getTags().put(key, e.getKey());
- break;
- }
- }
- }
- }
- }
-
- private static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
-
- @Override
- public boolean isKnownTag(String tag) {
- return tags.contains(tag) || tag.startsWith(SYSPROP);
- }
-
- private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
-
- List<String> requestedHostTags = new ArrayList<>();
- for (String tag : requestedTags) {
- if (IP_SNITCHES.contains(tag)) {
- requestedHostTags.add(tag);
- }
- }
-
- if (requestedHostTags.isEmpty()) {
- return;
- }
-
- String[] ipFragments = getIpFragments(solrNode);
-
- if (ipFragments == null) {
- return;
- }
-
- int ipSnitchCount = IP_SNITCHES.size();
- for (int i = 0; i < ipSnitchCount; i++) {
- String currentTagValue = ipFragments[i];
- String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
-
- if (requestedHostTags.contains(currentTagKey)) {
- context.getTags().put(currentTagKey, currentTagValue);
- }
- }
- }
-
- private String[] getIpFragments(String solrNode) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
- if (hostAndPortMatcher.find()) {
- String host = hostAndPortMatcher.group(1);
- if (host != null) {
- String ip = getHostIp(host);
- if (ip != null) {
- return ip.split(HOST_FRAG_SEPARATOR_REGEX); // IPv6 support will be provided by SOLR-8523
- }
- }
- }
-
- log.warn(
- "Failed to match host IP address from node URL [{}] using regex [{}]",
- solrNode,
- hostAndPortPattern.pattern());
- return null;
- }
-
- public String getHostIp(String host) {
- try {
- InetAddress address = InetAddress.getByName(host);
- return address.getHostAddress();
- } catch (Exception e) {
- log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
- return null;
- }
- }
-}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/Snitch.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/Snitch.java
deleted file mode 100644
index 916a639f647..00000000000
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/Snitch.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.cloud.rule;
-
-import java.util.Collections;
-import java.util.Set;
-
-public abstract class Snitch {
- public static final Set<Class<?>> WELL_KNOWN_SNITCHES =
- Collections.singleton(ImplicitSnitch.class);
-
- public abstract void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx);
-
- public abstract boolean isKnownTag(String tag);
-}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java
index e63660feb5f..038837fccf7 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java
@@ -16,31 +16,20 @@
*/
package org.apache.solr.common.cloud.rule;
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.util.Set;
-import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* This is the context provided to the snitches to interact with the system. This is a
* per-node-per-snitch instance.
*/
-public abstract class SnitchContext implements RemoteCallback {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+public abstract class SnitchContext {
private final Map<String, Object> tags = new HashMap<>();
private String node;
private Map<String, Object> session;
- public final SnitchInfo snitchInfo;
- public Exception exception;
- public SnitchContext(SnitchInfo perSnitch, String node, Map<String, Object> session) {
- this.snitchInfo = perSnitch;
+ public SnitchContext(String node, Map<String, Object> session) {
this.node = node;
this.session = session;
}
@@ -57,47 +46,9 @@ public abstract class SnitchContext implements RemoteCallback {
return session != null ? session.get(s) : null;
}
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return Collections.emptyMap();
- }
-
public abstract Map<?, ?> getZkJson(String path) throws KeeperException, InterruptedException;
public String getNode() {
return node;
}
-
- /**
- * make a call to solrnode/admin/cores with the given params and give a callback. This is designed
- * to be asynchronous because the system would want to batch the calls made to any given node
- *
- * @param node The node for which this call is made
- * @param params The params to be passed to the Snitch counterpart
- * @param klas The name of the class to be invoked in the remote node
- * @param callback The callback to be called when the response is obtained from remote node. If
- * this is passed as null the entire response map will be added as tags
- */
- @Deprecated
- public void invokeRemote(
- String node, ModifiableSolrParams params, String klas, RemoteCallback callback) {}
- ;
-
- @Override
- public void remoteCallback(SnitchContext ctx, Map<String, Object> returnedVal) {
- tags.putAll(returnedVal);
- }
-
- public String getErrMsg() {
- return exception == null ? null : exception.getMessage();
- }
-
- public abstract static class SnitchInfo {
- private final Map<String, Object> conf;
-
- protected SnitchInfo(Map<String, Object> conf) {
- this.conf = conf;
- }
-
- public abstract Set<String> getTagNames();
- }
}