You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:06:51 UTC

[lucene] 04/17: SOLR-15019: Add unit test for the collection metrics builders.

This is an automated email from the ASF dual-hosted git repository.

dweiss pushed a commit to branch jira/solr-15019
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit 4e7e82d5676706f99e35a8b663448699f9797fb4
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Dec 14 19:14:40 2020 +0100

    SOLR-15019: Add unit test for the collection metrics builders.
---
 .../placement/impl/AttributeFetcherImpl.java       |   2 +-
 .../placement/impl/CollectionMetricsBuilder.java   |  39 +++----
 .../cluster/placement/AttributeFetcherForTest.java |   4 +-
 .../apache/solr/cluster/placement/Builders.java    |  83 +++++++++++++-
 .../solr/cluster/placement/BuildersTest.java       | 126 +++++++++++++++++++++
 5 files changed, 221 insertions(+), 33 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
index 6a09836..b6bdfb6 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
@@ -211,7 +211,7 @@ public class AttributeFetcherImpl implements AttributeFetcher {
         nodeToHeapUsage,
         nodeToSystemLoadAverage,
         syspropSnitchToNodeToValue,
-        metricSnitchToNodeToValue);
+        metricSnitchToNodeToValue, null);
   }
 
   private static SolrInfoBean.Group getGroupFromMetricRegistry(NodeMetricRegistry registry) {
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
index eb29aa8..d3ce3db 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
@@ -31,30 +31,31 @@ public class CollectionMetricsBuilder {
 
   final Map<String, ShardMetricsBuilder> shardMetricsBuilders = new HashMap<>();
 
-  public void addShard(String shardName, ShardMetricsBuilder shardMetricsBuilder) {
+
+  public void addShardMetrics(String shardName, ShardMetricsBuilder shardMetricsBuilder) {
     shardMetricsBuilders.put(shardName, shardMetricsBuilder);
   }
 
   public CollectionMetrics build() {
     final Map<String, ShardMetrics> metricsMap = new HashMap<>();
     shardMetricsBuilders.forEach((shard, builder) -> metricsMap.put(shard, builder.build()));
-    return shardName -> {
-      if (metricsMap.containsKey(shardName)) {
-        return Optional.of(metricsMap.get(shardName));
-      } else {
-        return Optional.empty();
-      }
-    };
+    return shardName -> Optional.ofNullable(metricsMap.get(shardName));
   }
 
   public static class ShardMetricsBuilder {
     final Map<String, ReplicaMetricsBuilder> replicaMetricsBuilders = new HashMap<>();
 
-    public void addReplica(String replicaName, ReplicaMetricsBuilder replicaMetricsBuilder) {
+    public ShardMetricsBuilder addReplicaMetrics(String replicaName, ReplicaMetricsBuilder replicaMetricsBuilder) {
       replicaMetricsBuilders.put(replicaName, replicaMetricsBuilder);
+      return this;
+    }
+
+    public ShardMetricsBuilder setLeaderMetrics(ReplicaMetricsBuilder replicaMetricsBuilder) {
+      replicaMetricsBuilders.put(LEADER, replicaMetricsBuilder);
+      return this;
     }
 
-    private static final String LEADER = "__leader__";
+    public static final String LEADER = "__leader__";
 
     public ShardMetrics build() {
       final Map<String, ReplicaMetrics> metricsMap = new HashMap<>();
@@ -68,20 +69,12 @@ public class CollectionMetricsBuilder {
       return new ShardMetrics() {
         @Override
         public Optional<ReplicaMetrics> getLeaderMetrics() {
-          if (metricsMap.containsKey(LEADER)) {
-            return Optional.of(metricsMap.get(LEADER));
-          } else {
-            return Optional.empty();
-          }
+          return Optional.ofNullable(metricsMap.get(LEADER));
         }
 
         @Override
         public Optional<ReplicaMetrics> getReplicaMetrics(String replicaName) {
-          if (metricsMap.containsKey(replicaName)) {
-            return Optional.of(metricsMap.get(replicaName));
-          } else {
-            return Optional.empty();
-          }
+          return Optional.ofNullable(metricsMap.get(replicaName));
         }
       };
     }
@@ -116,11 +109,7 @@ public class CollectionMetricsBuilder {
 
         @Override
         public Optional<Object> getReplicaMetric(String metricName) {
-          if (metrics.containsKey(metricName)) {
-            return Optional.of(metrics.get(metricName));
-          } else {
-            return Optional.empty();
-          }
+          return Optional.ofNullable(metrics.get(metricName));
         }
       };
     }
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
index bb59317..268a354 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
@@ -19,8 +19,6 @@ package org.apache.solr.cluster.placement;
 
 import org.apache.solr.cluster.Node;
 import org.apache.solr.cluster.SolrCollection;
-import org.apache.solr.cluster.placement.AttributeFetcher;
-import org.apache.solr.cluster.placement.AttributeValues;
 
 import java.util.Set;
 
@@ -79,7 +77,7 @@ public class AttributeFetcherForTest implements AttributeFetcher {
 
   @Override
   public AttributeFetcher requestCollectionMetrics(SolrCollection solrCollection, Set<String> metricNames) {
-    throw new UnsupportedOperationException("Not yet implemented...");
+    return this;
   }
 
   @Override
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java b/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
index 9350a71..a903894 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
@@ -49,7 +49,11 @@ public class Builders {
     public ClusterBuilder initializeLiveNodes(int countNodes) {
       nodeBuilders = new LinkedList<>();
       for (int n = 0; n < countNodes; n++) {
-        nodeBuilders.add(new NodeBuilder().setNodeName("node_" + n)); // Default name, can be changed
+        NodeBuilder nodeBuilder = new NodeBuilder().setNodeName("node_" + n); // Default name, can be changed
+        nodeBuilder.setTotalDiskGB(10000L);
+        nodeBuilder.setFreeDiskGB(5000L);
+        nodeBuilder.setDiskType(AttributeFetcher.DiskHardwareType.SSD);
+        nodeBuilders.add(nodeBuilder);
       }
       return this;
     }
@@ -89,7 +93,9 @@ public class Builders {
 
     public AttributeFetcher buildAttributeFetcher() {
       Map<Node, Integer> nodeToCoreCount = new HashMap<>();
+      Map<Node, AttributeFetcher.DiskHardwareType> nodeToDiskType = new HashMap<>();
       Map<Node, Long> nodeToFreeDisk = new HashMap<>();
+      Map<Node, Long> nodeToTotalDisk = new HashMap<>();
       Map<String, Map<Node, String>> sysprops = new HashMap<>();
       Map<String, Map<Node, Double>> metrics = new HashMap<>();
       Map<String, CollectionMetrics> collectionMetrics = new HashMap<>();
@@ -101,12 +107,19 @@ public class Builders {
       for (NodeBuilder nodeBuilder : nodeBuilders) {
         Node node = nodeBuilder.build();
 
+        if (nodeBuilder.getDiskType() != null) {
+          nodeToDiskType.put(node, nodeBuilder.getDiskType());
+        }
+
         if (nodeBuilder.getCoreCount() != null) {
           nodeToCoreCount.put(node, nodeBuilder.getCoreCount());
         }
         if (nodeBuilder.getFreeDiskGB() != null) {
           nodeToFreeDisk.put(node, nodeBuilder.getFreeDiskGB());
         }
+        if (nodeBuilder.getTotalDiskGB() != null) {
+          nodeToTotalDisk.put(node, nodeBuilder.getTotalDiskGB());
+        }
         if (nodeBuilder.getSysprops() != null) {
           nodeBuilder.getSysprops().forEach((name, value) -> {
             sysprops.computeIfAbsent(name, n -> new HashMap<>())
@@ -123,10 +136,16 @@ public class Builders {
 
       collectionBuilders.forEach(builder -> {
         collectionMetrics.put(builder.collectionName, builder.collectionMetricsBuilder.build());
+        SolrCollection collection = builder.build();
+        collection.iterator().forEachRemaining(shard ->
+            shard.iterator().forEachRemaining(replica -> {
+                nodeToCoreCount.compute(replica.getNode(), (node, count) ->
+                    (count == null) ? 1 : count + 1);
+            }));
       });
 
-      AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk,
-          Map.of(), Map.of(), Map.of(), sysprops, metrics, collectionMetrics);
+      AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, nodeToDiskType, nodeToFreeDisk,
+          nodeToTotalDisk, Map.of(), Map.of(), sysprops, metrics, collectionMetrics);
       return new AttributeFetcherForTest(attributeValues);
     }
   }
@@ -234,18 +253,46 @@ public class Builders {
      * Initializes shard and replica builders for the collection based on passed parameters. Replicas are assigned round
      * robin to the nodes. The shard leader is the first NRT replica of each shard (or first TLOG is no NRT).
      * Shard and replica configuration can be modified afterwards, the returned builder hierarchy is a convenient starting point.
+     * @param countShards number of shards to create
+     * @param countNrtReplicas number of NRT replicas per shard
+     * @param countTlogReplicas number of TLOG replicas per shard
+     * @param countPullReplicas number of PULL replicas per shard
+     * @param nodes list of nodes to place replicas on.
+     */
+    public CollectionBuilder initializeShardsReplicas(int countShards, int countNrtReplicas, int countTlogReplicas,
+                                                      int countPullReplicas, List<NodeBuilder> nodes) {
+      return initializeShardsReplicas(countShards, countNrtReplicas, countTlogReplicas, countPullReplicas, nodes, null);
+    }
+
+    /**
+     * Initializes shard and replica builders for the collection based on passed parameters. Replicas are assigned round
+     * robin to the nodes. The shard leader is the first NRT replica of each shard (or first TLOG is no NRT).
+     * Shard and replica configuration can be modified afterwards, the returned builder hierarchy is a convenient starting point.
+     * @param countShards number of shards to create
+     * @param countNrtReplicas number of NRT replicas per shard
+     * @param countTlogReplicas number of TLOG replicas per shard
+     * @param countPullReplicas number of PULL replicas per shard
+     * @param nodes list of nodes to place replicas on.
+     * @param initialSizeGBPerShard initial replica size (in GB) per shard
      */
     public CollectionBuilder initializeShardsReplicas(int countShards, int countNrtReplicas, int countTlogReplicas,
-                                               int countPullReplicas, List<NodeBuilder> nodes) {
+                                               int countPullReplicas, List<NodeBuilder> nodes,
+                                                      List<Integer> initialSizeGBPerShard) {
       Iterator<NodeBuilder> nodeIterator = nodes.iterator();
 
       shardBuilders = new LinkedList<>();
+      if (initialSizeGBPerShard != null && initialSizeGBPerShard.size() != countShards) {
+        throw new RuntimeException("list of shard sizes must be the same length as the countShards!");
+      }
 
       for (int shardNumber = 1; shardNumber <= countShards; shardNumber++) {
         String shardName = buildShardName(shardNumber);
 
+        CollectionMetricsBuilder.ShardMetricsBuilder shardMetricsBuilder = new CollectionMetricsBuilder.ShardMetricsBuilder();
+
         LinkedList<ReplicaBuilder> replicas = new LinkedList<>();
         ReplicaBuilder leader = null;
+        CollectionMetricsBuilder.ReplicaMetricsBuilder leaderMetrics = null;
 
         // Iterate on requested counts, NRT then TLOG then PULL. Leader chosen as first NRT (or first TLOG if no NRT)
         List<Pair<Replica.ReplicaType, Integer>> replicaTypes = List.of(
@@ -270,15 +317,23 @@ public class Builders {
                 .setReplicaState(Replica.ReplicaState.ACTIVE).setReplicaNode(node);
             replicas.add(replicaBuilder);
 
+            CollectionMetricsBuilder.ReplicaMetricsBuilder replicaMetricsBuilder = new CollectionMetricsBuilder.ReplicaMetricsBuilder();
+            shardMetricsBuilder.addReplicaMetrics(replicaName, replicaMetricsBuilder);
+            if (initialSizeGBPerShard != null) {
+              replicaMetricsBuilder.setSizeGB(initialSizeGBPerShard.get(shardNumber - 1));
+            }
             if (leader == null && type != Replica.ReplicaType.PULL) {
               leader = replicaBuilder;
+              leaderMetrics = replicaMetricsBuilder;
             }
           }
         }
 
         ShardBuilder shardBuilder = new ShardBuilder();
         shardBuilder.setShardName(shardName).setReplicaBuilders(replicas).setLeader(leader);
+        shardMetricsBuilder.setLeaderMetrics(leaderMetrics);
         shardBuilders.add(shardBuilder);
+        collectionMetricsBuilder.addShardMetrics(shardName, shardMetricsBuilder);
       }
 
       return this;
@@ -404,6 +459,8 @@ public class Builders {
     private String nodeName = null;
     private Integer coreCount = null;
     private Long freeDiskGB = null;
+    private Long totalDiskGB = null;
+    private AttributeFetcher.DiskHardwareType diskType;
     private Map<String, String> sysprops = null;
     private Map<String, Double> metrics = null;
 
@@ -422,6 +479,11 @@ public class Builders {
       return this;
     }
 
+    public NodeBuilder setTotalDiskGB(Long totalDiskGB) {
+      this.totalDiskGB = totalDiskGB;
+      return this;
+    }
+
     public NodeBuilder setSysprop(String key, String value) {
       if (sysprops == null) {
         sysprops = new HashMap<>();
@@ -431,6 +493,11 @@ public class Builders {
       return this;
     }
 
+    public NodeBuilder setDiskType(AttributeFetcher.DiskHardwareType diskType) {
+      this.diskType = diskType;
+      return this;
+    }
+
     public NodeBuilder setMetric(AttributeFetcher.NodeMetricRegistry registry, String key, Double value) {
       if (metrics == null) {
         metrics = new HashMap<>();
@@ -448,6 +515,14 @@ public class Builders {
       return freeDiskGB;
     }
 
+    public Long getTotalDiskGB() {
+      return totalDiskGB;
+    }
+
+    public AttributeFetcher.DiskHardwareType getDiskType() {
+      return diskType;
+    }
+
     public Map<String, String> getSysprops() {
       return sysprops;
     }
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java
new file mode 100644
index 0000000..57cac4b
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cluster.placement;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cluster.Cluster;
+import org.apache.solr.cluster.Node;
+import org.apache.solr.cluster.Shard;
+import org.apache.solr.cluster.SolrCollection;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.solr.cluster.placement.Builders.*;
+
+/**
+ *
+ */
+public class BuildersTest extends SolrTestCaseJ4 {
+
+  @Test
+  public void testClusterBuilder() throws Exception {
+    int NUM_NODES = 3;
+    int NUM_SHARDS = 2;
+    int NUM_NRT_REPLICAS = 2;
+    String collectionName = "test";
+    ClusterBuilder clusterBuilder = newClusterBuilder()
+        .initializeLiveNodes(NUM_NODES);
+    CollectionBuilder collectionBuilder = newCollectionBuilder(collectionName)
+        .initializeShardsReplicas(NUM_SHARDS,
+            NUM_NRT_REPLICAS,
+            NUM_NRT_REPLICAS + 1,
+            NUM_NRT_REPLICAS + 2,
+            clusterBuilder.getLiveNodeBuilders(),
+            List.of(10, 20));
+    clusterBuilder.addCollection(collectionBuilder);
+    Cluster cluster = clusterBuilder.build();
+    assertEquals("number of nodes", NUM_NODES, cluster.getLiveNodes().size());
+    SolrCollection collection = cluster.getCollection(collectionName);
+    assertNotNull("collection", collection);
+    assertEquals("shards", 2, collection.getShardNames().size());
+    for (String shardName : collection.getShardNames()) {
+      Shard shard = collection.getShard(shardName);
+      assertNotNull("shard leader", shard.getLeader());
+      int[] counts = new int[3];
+      shard.iterator().forEachRemaining(r -> {
+        switch (r.getType()) {
+          case NRT:
+            counts[0]++;
+            break;
+          case TLOG:
+            counts[1]++;
+            break;
+          case PULL:
+            counts[2]++;
+        }
+      });
+      assertEquals("numNrt", NUM_NRT_REPLICAS, counts[0]);
+      assertEquals("numTlog", NUM_NRT_REPLICAS + 1, counts[1]);
+      assertEquals("numPull", NUM_NRT_REPLICAS + 2, counts[2]);
+    }
+    // AttributeFetcher
+    AttributeFetcher attributeFetcher = clusterBuilder.buildAttributeFetcher();
+    attributeFetcher
+        .fetchFrom(cluster.getLiveNodes())
+        .requestNodeCoreCount()
+        .requestNodeDiskType()
+        .requestNodeFreeDisk()
+        .requestNodeTotalDisk()
+        .requestCollectionMetrics(collection, Set.of());
+    AttributeValues attributeValues = attributeFetcher.fetchAttributes();
+    for (Node node : cluster.getLiveNodes()) {
+      Optional<Integer> coreCount = attributeValues.getCoresCount(node);
+      assertTrue("coreCount present", coreCount.isPresent());
+      Optional<AttributeFetcher.DiskHardwareType> diskType = attributeValues.getDiskType(node);
+      assertTrue("diskType present", diskType.isPresent());
+      Optional<Long> diskOpt = attributeValues.getFreeDisk(node);
+      assertTrue("freeDisk", diskOpt.isPresent());
+      diskOpt = attributeValues.getTotalDisk(node);
+      assertTrue("totalDisk", diskOpt.isPresent());
+    }
+    Optional<CollectionMetrics> collectionMetricsOpt = attributeValues.getCollectionMetrics(collectionName);
+    assertTrue("collectionMetrics present", collectionMetricsOpt.isPresent());
+    CollectionMetrics collectionMetrics = collectionMetricsOpt.get();
+    for (String shardName : collection.getShardNames()) {
+      Optional<ShardMetrics> shardMetricsOpt = collectionMetrics.getShardMetrics(shardName);
+      assertTrue("shard metrics", shardMetricsOpt.isPresent());
+      ShardMetrics shardMetrics = shardMetricsOpt.get();
+      Optional<ReplicaMetrics> replicaMetricsOpt = shardMetrics.getLeaderMetrics();
+      assertTrue("leader metrics", replicaMetricsOpt.isPresent());
+      ReplicaMetrics leaderMetrics = replicaMetricsOpt.get();
+      if (shardName.endsWith("1")) {
+        assertEquals("size", 10, leaderMetrics.getReplicaSizeGB());
+      } else {
+        assertEquals("size", 20, leaderMetrics.getReplicaSizeGB());
+      }
+      Shard shard = collection.getShard(shardName);
+      shard.iterator().forEachRemaining(r -> {
+        Optional<ReplicaMetrics> metricsOpt = shardMetrics.getReplicaMetrics(r.getReplicaName());
+        assertTrue("replica metrics", metricsOpt.isPresent());
+        ReplicaMetrics metrics = metricsOpt.get();
+        if (shardName.endsWith("1")) {
+          assertEquals("size", 10, metrics.getReplicaSizeGB());
+        } else {
+          assertEquals("size", 20, metrics.getReplicaSizeGB());
+        }
+      });
+    }
+  }
+}