You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2024/06/05 11:54:21 UTC
(solr) branch branch_9x updated: SOLR-17099: snitch does not return spurious tags (#2278)
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 99dcb2be98d SOLR-17099: snitch does not return spurious tags (#2278)
99dcb2be98d is described below
commit 99dcb2be98d4ef4dcea6aec0de084abf19b06a94
Author: Pierre Salagnac <82...@users.noreply.github.com>
AuthorDate: Wed Jun 5 13:51:15 2024 +0200
SOLR-17099: snitch does not return spurious tags (#2278)
(cherry picked from commit d1a4cbe487a001cea790ca35d0479b0ed7c0ebcd)
---
solr/CHANGES.txt | 3 +-
.../solr/client/solrj/impl/NodeValueFetcher.java | 61 ++++++++++-----
.../solrj/impl/SolrClientNodeStateProvider.java | 4 +-
.../client/solrj/impl/NodeValueFetcherTest.java | 89 ++++++++++++++++++++++
4 files changed, 135 insertions(+), 22 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e4d0f586ad7..f57bb243386 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -36,6 +36,8 @@ Optimizations
* SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather
cluster metrics. AttributeFetcherImpl was improved accordingly. (Yohann Callea)
+* SOLR-17099: Do not return spurious tags when fetching metrics from a Solr node to another. (Pierre Salagnac)
+
Bug Fixes
---------------------
* SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh)
@@ -134,7 +136,6 @@ Optimizations
* GITHUB#2217: Scale to 10K+ collections better in ZkStateReader.refreshCollectionsList (David Smiley)
-
Bug Fixes
---------------------
* SOLR-17153: CloudSolrClient could fail a request immediately following a collection creation. Required double-checking the collection doesn’t exist. (Aparna Suresh via David Smiley)
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
index c6e122aae50..56ec552723d 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
@@ -18,6 +18,7 @@
package org.apache.solr.client.solrj.impl;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -102,11 +103,12 @@ public class NodeValueFetcher {
}
}
- private void getRemoteInfo(
- String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
- if (!(ctx).isNodeAlive(solrNode)) return;
+ /** Retrieve values that match JVM system properties and metrics. */
+ private void getRemotePropertiesAndMetrics(
+ Set<String> requestedTagNames, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+
Map<String, Set<Object>> metricsKeyVsTag = new HashMap<>();
- for (String tag : requestedTags) {
+ for (String tag : requestedTagNames) {
if (tag.startsWith(SYSPROP)) {
metricsKeyVsTag
.computeIfAbsent(
@@ -120,28 +122,42 @@ public class NodeValueFetcher {
}
}
if (!metricsKeyVsTag.isEmpty()) {
- SolrClientNodeStateProvider.fetchReplicaMetrics(solrNode, ctx, metricsKeyVsTag);
+ SolrClientNodeStateProvider.fetchReplicaMetrics(ctx.getNode(), ctx, metricsKeyVsTag);
}
+ }
- Set<String> groups = new HashSet<>();
- List<String> prefixes = new ArrayList<>();
+ /** Retrieve values of well known tags, as defined in {@link Tags}. */
+ private void getRemoteTags(
+ Set<String> requestedTagNames, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+
+ // First resolve names into actual Tags instances
+ EnumSet<Tags> requestedTags = EnumSet.noneOf(Tags.class);
for (Tags t : Tags.values()) {
- if (requestedTags.contains(t.tagName)) {
- groups.add(t.group);
- prefixes.add(t.prefix);
+ if (requestedTagNames.contains(t.tagName)) {
+ requestedTags.add(t);
}
}
- if (groups.isEmpty() || prefixes.isEmpty()) return;
+ if (requestedTags.isEmpty()) {
+ return;
+ }
+
+ Set<String> groups = new HashSet<>();
+ List<String> prefixes = new ArrayList<>();
+ for (Tags t : requestedTags) {
+ groups.add(t.group);
+ prefixes.add(t.prefix);
+ }
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("group", StrUtils.join(groups, ','));
params.add("prefix", StrUtils.join(prefixes, ','));
try {
- SimpleSolrResponse rsp = ctx.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);
+ SimpleSolrResponse rsp =
+ ctx.invokeWithRetry(ctx.getNode(), CommonParams.METRICS_PATH, params);
NamedList<?> metrics = (NamedList<?>) rsp.getResponse().get("metrics");
if (metrics != null) {
- for (Tags t : Tags.values()) {
+ for (Tags t : requestedTags) {
ctx.tags.put(t.tagName, t.extractResult(metrics));
}
}
@@ -150,19 +166,26 @@ public class NodeValueFetcher {
}
}
- public void getTags(
- String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+ public void getTags(Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+
try {
- if (requestedTags.contains(NODE)) ctx.tags.put(NODE, solrNode);
+ if (requestedTags.contains(NODE)) ctx.tags.put(NODE, ctx.getNode());
if (requestedTags.contains(HOST)) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(ctx.getNode());
if (hostAndPortMatcher.find()) ctx.tags.put(HOST, hostAndPortMatcher.group(1));
}
if (requestedTags.contains(PORT)) {
- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(ctx.getNode());
if (hostAndPortMatcher.find()) ctx.tags.put(PORT, hostAndPortMatcher.group(2));
}
- getRemoteInfo(solrNode, requestedTags, ctx);
+
+ if (!ctx.isNodeAlive(ctx.getNode())) {
+ // Don't try to reach out to the node if we already know it is down
+ return;
+ }
+
+ getRemotePropertiesAndMetrics(requestedTags, ctx);
+ getRemoteTags(requestedTags, ctx);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 4d7f087426a..7e70378c582 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -115,7 +115,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
protected Map<String, Object> fetchTagValues(String node, Collection<String> tags) {
NodeValueFetcher nodeValueFetcher = new NodeValueFetcher();
RemoteCallCtx ctx = new RemoteCallCtx(node, solrClient);
- nodeValueFetcher.getTags(node, new HashSet<>(tags), ctx);
+ nodeValueFetcher.getTags(new HashSet<>(tags), ctx);
return ctx.tags;
}
@@ -228,7 +228,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
ZkClientClusterStateProvider zkClientClusterStateProvider;
CloudLegacySolrClient solrClient;
public final Map<String, Object> tags = new HashMap<>();
- private String node;
+ private final String node;
public Map<String, Object> session;
public boolean isNodeAlive(String node) {
diff --git a/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java b/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java
new file mode 100644
index 00000000000..5f521dbb20c
--- /dev/null
+++ b/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.util.Set;
+import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.RemoteCallCtx;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class NodeValueFetcherTest extends SolrCloudTestCase {
+
+ private static final String COLLECTION =
+ NodeValueFetcherTest.class.getSimpleName() + "_collection";
+
+ private static int numShards;
+ private static int numReplicas;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ // Metrics should be enabled to get them with snitch
+ System.setProperty("metricsEnabled", "true");
+
+ numShards = random().nextInt(3) + 3;
+ numReplicas = random().nextInt(2) + 2;
+ int numNodes = random().nextInt(2) + 1;
+
+ configureCluster(numNodes).configure();
+ }
+
+ @Before
+ public void setupCollection() throws Exception {
+ CollectionAdminResponse rsp =
+ CollectionAdminRequest.createCollection(COLLECTION, numShards, numReplicas)
+ .process(cluster.getSolrClient());
+ assertTrue(rsp.isSuccess());
+ cluster.waitForActiveCollection(COLLECTION, numShards, numShards * numReplicas);
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ cluster.deleteAllCollections();
+ }
+
+ @Test
+ public void testGetTags() {
+
+ CloudLegacySolrClient solrClient = (CloudLegacySolrClient) cluster.getSolrClient();
+ int totalCores = 0;
+
+ // Sum all the cores of the collection by fetching tags of all nodes.
+ // We should get same number than when we created the collection
+ for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
+ String node = runner.getNodeName();
+ RemoteCallCtx ctx = new RemoteCallCtx(node, solrClient);
+ NodeValueFetcher fetcher = new NodeValueFetcher();
+
+ Set<String> requestedTags = Set.of("cores");
+ fetcher.getTags(requestedTags, ctx);
+
+ // make sure we only get the tag we asked
+ assertEquals(1, ctx.tags.size());
+
+ int coresOnNode = (Integer) ctx.tags.get("cores");
+ totalCores += coresOnNode;
+ }
+
+ assertEquals(numShards * numReplicas, totalCores);
+ }
+}