You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2024/06/05 11:54:21 UTC

(solr) branch branch_9x updated: SOLR-17099: snitch does not return spurious tags (#2278)

This is an automated email from the ASF dual-hosted git repository.

janhoy pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new 99dcb2be98d SOLR-17099: snitch does not return spurious tags (#2278)
99dcb2be98d is described below

commit 99dcb2be98d4ef4dcea6aec0de084abf19b06a94
Author: Pierre Salagnac <82...@users.noreply.github.com>
AuthorDate: Wed Jun 5 13:51:15 2024 +0200

    SOLR-17099: snitch does not return spurious tags (#2278)
    
    (cherry picked from commit d1a4cbe487a001cea790ca35d0479b0ed7c0ebcd)
---
 solr/CHANGES.txt                                   |  3 +-
 .../solr/client/solrj/impl/NodeValueFetcher.java   | 61 ++++++++++-----
 .../solrj/impl/SolrClientNodeStateProvider.java    |  4 +-
 .../client/solrj/impl/NodeValueFetcherTest.java    | 89 ++++++++++++++++++++++
 4 files changed, 135 insertions(+), 22 deletions(-)

diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e4d0f586ad7..f57bb243386 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -36,6 +36,8 @@ Optimizations
 * SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather
   cluster metrics.  AttributeFetcherImpl was improved accordingly.  (Yohann Callea)
 
+* SOLR-17099: Do not return spurious tags when fetching metrics from a Solr node to another. (Pierre Salagnac)
+
 Bug Fixes
 ---------------------
 * SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh)
@@ -134,7 +136,6 @@ Optimizations
 
 * GITHUB#2217: Scale to 10K+ collections better in ZkStateReader.refreshCollectionsList (David Smiley)
 
-
 Bug Fixes
 ---------------------
 * SOLR-17153: CloudSolrClient could fail a request immediately following a collection creation.  Required double-checking the collection doesn’t exist. (Aparna Suresh via David Smiley)
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
index c6e122aae50..56ec552723d 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java
@@ -18,6 +18,7 @@
 package org.apache.solr.client.solrj.impl;
 
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -102,11 +103,12 @@ public class NodeValueFetcher {
     }
   }
 
-  private void getRemoteInfo(
-      String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
-    if (!(ctx).isNodeAlive(solrNode)) return;
+  /** Retrieve values that match JVM system properties and metrics. */
+  private void getRemotePropertiesAndMetrics(
+      Set<String> requestedTagNames, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+
     Map<String, Set<Object>> metricsKeyVsTag = new HashMap<>();
-    for (String tag : requestedTags) {
+    for (String tag : requestedTagNames) {
       if (tag.startsWith(SYSPROP)) {
         metricsKeyVsTag
             .computeIfAbsent(
@@ -120,28 +122,42 @@ public class NodeValueFetcher {
       }
     }
     if (!metricsKeyVsTag.isEmpty()) {
-      SolrClientNodeStateProvider.fetchReplicaMetrics(solrNode, ctx, metricsKeyVsTag);
+      SolrClientNodeStateProvider.fetchReplicaMetrics(ctx.getNode(), ctx, metricsKeyVsTag);
     }
+  }
 
-    Set<String> groups = new HashSet<>();
-    List<String> prefixes = new ArrayList<>();
+  /** Retrieve values of well known tags, as defined in {@link Tags}. */
+  private void getRemoteTags(
+      Set<String> requestedTagNames, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+
+    // First resolve names into actual Tags instances
+    EnumSet<Tags> requestedTags = EnumSet.noneOf(Tags.class);
     for (Tags t : Tags.values()) {
-      if (requestedTags.contains(t.tagName)) {
-        groups.add(t.group);
-        prefixes.add(t.prefix);
+      if (requestedTagNames.contains(t.tagName)) {
+        requestedTags.add(t);
       }
     }
-    if (groups.isEmpty() || prefixes.isEmpty()) return;
+    if (requestedTags.isEmpty()) {
+      return;
+    }
+
+    Set<String> groups = new HashSet<>();
+    List<String> prefixes = new ArrayList<>();
+    for (Tags t : requestedTags) {
+      groups.add(t.group);
+      prefixes.add(t.prefix);
+    }
 
     ModifiableSolrParams params = new ModifiableSolrParams();
     params.add("group", StrUtils.join(groups, ','));
     params.add("prefix", StrUtils.join(prefixes, ','));
 
     try {
-      SimpleSolrResponse rsp = ctx.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);
+      SimpleSolrResponse rsp =
+          ctx.invokeWithRetry(ctx.getNode(), CommonParams.METRICS_PATH, params);
       NamedList<?> metrics = (NamedList<?>) rsp.getResponse().get("metrics");
       if (metrics != null) {
-        for (Tags t : Tags.values()) {
+        for (Tags t : requestedTags) {
           ctx.tags.put(t.tagName, t.extractResult(metrics));
         }
       }
@@ -150,19 +166,26 @@ public class NodeValueFetcher {
     }
   }
 
-  public void getTags(
-      String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+  public void getTags(Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {
+
     try {
-      if (requestedTags.contains(NODE)) ctx.tags.put(NODE, solrNode);
+      if (requestedTags.contains(NODE)) ctx.tags.put(NODE, ctx.getNode());
       if (requestedTags.contains(HOST)) {
-        Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+        Matcher hostAndPortMatcher = hostAndPortPattern.matcher(ctx.getNode());
         if (hostAndPortMatcher.find()) ctx.tags.put(HOST, hostAndPortMatcher.group(1));
       }
       if (requestedTags.contains(PORT)) {
-        Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+        Matcher hostAndPortMatcher = hostAndPortPattern.matcher(ctx.getNode());
         if (hostAndPortMatcher.find()) ctx.tags.put(PORT, hostAndPortMatcher.group(2));
       }
-      getRemoteInfo(solrNode, requestedTags, ctx);
+
+      if (!ctx.isNodeAlive(ctx.getNode())) {
+        // Don't try to reach out to the node if we already know it is down
+        return;
+      }
+
+      getRemotePropertiesAndMetrics(requestedTags, ctx);
+      getRemoteTags(requestedTags, ctx);
     } catch (Exception e) {
       throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
     }
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 4d7f087426a..7e70378c582 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -115,7 +115,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
   protected Map<String, Object> fetchTagValues(String node, Collection<String> tags) {
     NodeValueFetcher nodeValueFetcher = new NodeValueFetcher();
     RemoteCallCtx ctx = new RemoteCallCtx(node, solrClient);
-    nodeValueFetcher.getTags(node, new HashSet<>(tags), ctx);
+    nodeValueFetcher.getTags(new HashSet<>(tags), ctx);
     return ctx.tags;
   }
 
@@ -228,7 +228,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
     ZkClientClusterStateProvider zkClientClusterStateProvider;
     CloudLegacySolrClient solrClient;
     public final Map<String, Object> tags = new HashMap<>();
-    private String node;
+    private final String node;
     public Map<String, Object> session;
 
     public boolean isNodeAlive(String node) {
diff --git a/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java b/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java
new file mode 100644
index 00000000000..5f521dbb20c
--- /dev/null
+++ b/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.util.Set;
+import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.RemoteCallCtx;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class NodeValueFetcherTest extends SolrCloudTestCase {
+
+  private static final String COLLECTION =
+      NodeValueFetcherTest.class.getSimpleName() + "_collection";
+
+  private static int numShards;
+  private static int numReplicas;
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    // Metrics should be enabled to get them with snitch
+    System.setProperty("metricsEnabled", "true");
+
+    numShards = random().nextInt(3) + 3;
+    numReplicas = random().nextInt(2) + 2;
+    int numNodes = random().nextInt(2) + 1;
+
+    configureCluster(numNodes).configure();
+  }
+
+  @Before
+  public void setupCollection() throws Exception {
+    CollectionAdminResponse rsp =
+        CollectionAdminRequest.createCollection(COLLECTION, numShards, numReplicas)
+            .process(cluster.getSolrClient());
+    assertTrue(rsp.isSuccess());
+    cluster.waitForActiveCollection(COLLECTION, numShards, numShards * numReplicas);
+  }
+
+  @After
+  public void cleanup() throws Exception {
+    cluster.deleteAllCollections();
+  }
+
+  @Test
+  public void testGetTags() {
+
+    CloudLegacySolrClient solrClient = (CloudLegacySolrClient) cluster.getSolrClient();
+    int totalCores = 0;
+
+    // Sum all the cores of the collection by fetching tags of all nodes.
+    // We should get same number than when we created the collection
+    for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
+      String node = runner.getNodeName();
+      RemoteCallCtx ctx = new RemoteCallCtx(node, solrClient);
+      NodeValueFetcher fetcher = new NodeValueFetcher();
+
+      Set<String> requestedTags = Set.of("cores");
+      fetcher.getTags(requestedTags, ctx);
+
+      // make sure we only get the tag we asked
+      assertEquals(1, ctx.tags.size());
+
+      int coresOnNode = (Integer) ctx.tags.get("cores");
+      totalCores += coresOnNode;
+    }
+
+    assertEquals(numShards * numReplicas, totalCores);
+  }
+}