You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:06:51 UTC
[lucene] 04/17: SOLR-15019: Add unit test for the collection
metrics builders.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-15019
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 4e7e82d5676706f99e35a8b663448699f9797fb4
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Dec 14 19:14:40 2020 +0100
SOLR-15019: Add unit test for the collection metrics builders.
---
.../placement/impl/AttributeFetcherImpl.java | 2 +-
.../placement/impl/CollectionMetricsBuilder.java | 39 +++----
.../cluster/placement/AttributeFetcherForTest.java | 4 +-
.../apache/solr/cluster/placement/Builders.java | 83 +++++++++++++-
.../solr/cluster/placement/BuildersTest.java | 126 +++++++++++++++++++++
5 files changed, 221 insertions(+), 33 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
index 6a09836..b6bdfb6 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/AttributeFetcherImpl.java
@@ -211,7 +211,7 @@ public class AttributeFetcherImpl implements AttributeFetcher {
nodeToHeapUsage,
nodeToSystemLoadAverage,
syspropSnitchToNodeToValue,
- metricSnitchToNodeToValue);
+ metricSnitchToNodeToValue, null);
}
private static SolrInfoBean.Group getGroupFromMetricRegistry(NodeMetricRegistry registry) {
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
index eb29aa8..d3ce3db 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/CollectionMetricsBuilder.java
@@ -31,30 +31,31 @@ public class CollectionMetricsBuilder {
final Map<String, ShardMetricsBuilder> shardMetricsBuilders = new HashMap<>();
- public void addShard(String shardName, ShardMetricsBuilder shardMetricsBuilder) {
+
+ public void addShardMetrics(String shardName, ShardMetricsBuilder shardMetricsBuilder) {
shardMetricsBuilders.put(shardName, shardMetricsBuilder);
}
public CollectionMetrics build() {
final Map<String, ShardMetrics> metricsMap = new HashMap<>();
shardMetricsBuilders.forEach((shard, builder) -> metricsMap.put(shard, builder.build()));
- return shardName -> {
- if (metricsMap.containsKey(shardName)) {
- return Optional.of(metricsMap.get(shardName));
- } else {
- return Optional.empty();
- }
- };
+ return shardName -> Optional.ofNullable(metricsMap.get(shardName));
}
public static class ShardMetricsBuilder {
final Map<String, ReplicaMetricsBuilder> replicaMetricsBuilders = new HashMap<>();
- public void addReplica(String replicaName, ReplicaMetricsBuilder replicaMetricsBuilder) {
+ public ShardMetricsBuilder addReplicaMetrics(String replicaName, ReplicaMetricsBuilder replicaMetricsBuilder) {
replicaMetricsBuilders.put(replicaName, replicaMetricsBuilder);
+ return this;
+ }
+
+ public ShardMetricsBuilder setLeaderMetrics(ReplicaMetricsBuilder replicaMetricsBuilder) {
+ replicaMetricsBuilders.put(LEADER, replicaMetricsBuilder);
+ return this;
}
- private static final String LEADER = "__leader__";
+ public static final String LEADER = "__leader__";
public ShardMetrics build() {
final Map<String, ReplicaMetrics> metricsMap = new HashMap<>();
@@ -68,20 +69,12 @@ public class CollectionMetricsBuilder {
return new ShardMetrics() {
@Override
public Optional<ReplicaMetrics> getLeaderMetrics() {
- if (metricsMap.containsKey(LEADER)) {
- return Optional.of(metricsMap.get(LEADER));
- } else {
- return Optional.empty();
- }
+ return Optional.ofNullable(metricsMap.get(LEADER));
}
@Override
public Optional<ReplicaMetrics> getReplicaMetrics(String replicaName) {
- if (metricsMap.containsKey(replicaName)) {
- return Optional.of(metricsMap.get(replicaName));
- } else {
- return Optional.empty();
- }
+ return Optional.ofNullable(metricsMap.get(replicaName));
}
};
}
@@ -116,11 +109,7 @@ public class CollectionMetricsBuilder {
@Override
public Optional<Object> getReplicaMetric(String metricName) {
- if (metrics.containsKey(metricName)) {
- return Optional.of(metrics.get(metricName));
- } else {
- return Optional.empty();
- }
+ return Optional.ofNullable(metrics.get(metricName));
}
};
}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
index bb59317..268a354 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/AttributeFetcherForTest.java
@@ -19,8 +19,6 @@ package org.apache.solr.cluster.placement;
import org.apache.solr.cluster.Node;
import org.apache.solr.cluster.SolrCollection;
-import org.apache.solr.cluster.placement.AttributeFetcher;
-import org.apache.solr.cluster.placement.AttributeValues;
import java.util.Set;
@@ -79,7 +77,7 @@ public class AttributeFetcherForTest implements AttributeFetcher {
@Override
public AttributeFetcher requestCollectionMetrics(SolrCollection solrCollection, Set<String> metricNames) {
- throw new UnsupportedOperationException("Not yet implemented...");
+ return this;
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java b/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
index 9350a71..a903894 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/Builders.java
@@ -49,7 +49,11 @@ public class Builders {
public ClusterBuilder initializeLiveNodes(int countNodes) {
nodeBuilders = new LinkedList<>();
for (int n = 0; n < countNodes; n++) {
- nodeBuilders.add(new NodeBuilder().setNodeName("node_" + n)); // Default name, can be changed
+ NodeBuilder nodeBuilder = new NodeBuilder().setNodeName("node_" + n); // Default name, can be changed
+ nodeBuilder.setTotalDiskGB(10000L);
+ nodeBuilder.setFreeDiskGB(5000L);
+ nodeBuilder.setDiskType(AttributeFetcher.DiskHardwareType.SSD);
+ nodeBuilders.add(nodeBuilder);
}
return this;
}
@@ -89,7 +93,9 @@ public class Builders {
public AttributeFetcher buildAttributeFetcher() {
Map<Node, Integer> nodeToCoreCount = new HashMap<>();
+ Map<Node, AttributeFetcher.DiskHardwareType> nodeToDiskType = new HashMap<>();
Map<Node, Long> nodeToFreeDisk = new HashMap<>();
+ Map<Node, Long> nodeToTotalDisk = new HashMap<>();
Map<String, Map<Node, String>> sysprops = new HashMap<>();
Map<String, Map<Node, Double>> metrics = new HashMap<>();
Map<String, CollectionMetrics> collectionMetrics = new HashMap<>();
@@ -101,12 +107,19 @@ public class Builders {
for (NodeBuilder nodeBuilder : nodeBuilders) {
Node node = nodeBuilder.build();
+ if (nodeBuilder.getDiskType() != null) {
+ nodeToDiskType.put(node, nodeBuilder.getDiskType());
+ }
+
if (nodeBuilder.getCoreCount() != null) {
nodeToCoreCount.put(node, nodeBuilder.getCoreCount());
}
if (nodeBuilder.getFreeDiskGB() != null) {
nodeToFreeDisk.put(node, nodeBuilder.getFreeDiskGB());
}
+ if (nodeBuilder.getTotalDiskGB() != null) {
+ nodeToTotalDisk.put(node, nodeBuilder.getTotalDiskGB());
+ }
if (nodeBuilder.getSysprops() != null) {
nodeBuilder.getSysprops().forEach((name, value) -> {
sysprops.computeIfAbsent(name, n -> new HashMap<>())
@@ -123,10 +136,16 @@ public class Builders {
collectionBuilders.forEach(builder -> {
collectionMetrics.put(builder.collectionName, builder.collectionMetricsBuilder.build());
+ SolrCollection collection = builder.build();
+ collection.iterator().forEachRemaining(shard ->
+ shard.iterator().forEachRemaining(replica -> {
+ nodeToCoreCount.compute(replica.getNode(), (node, count) ->
+ (count == null) ? 1 : count + 1);
+ }));
});
- AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk,
- Map.of(), Map.of(), Map.of(), sysprops, metrics, collectionMetrics);
+ AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, nodeToDiskType, nodeToFreeDisk,
+ nodeToTotalDisk, Map.of(), Map.of(), sysprops, metrics, collectionMetrics);
return new AttributeFetcherForTest(attributeValues);
}
}
@@ -234,18 +253,46 @@ public class Builders {
* Initializes shard and replica builders for the collection based on passed parameters. Replicas are assigned round
* robin to the nodes. The shard leader is the first NRT replica of each shard (or first TLOG is no NRT).
* Shard and replica configuration can be modified afterwards, the returned builder hierarchy is a convenient starting point.
+ * @param countShards number of shards to create
+ * @param countNrtReplicas number of NRT replicas per shard
+ * @param countTlogReplicas number of TLOG replicas per shard
+ * @param countPullReplicas number of PULL replicas per shard
+ * @param nodes list of nodes to place replicas on.
+ */
+ public CollectionBuilder initializeShardsReplicas(int countShards, int countNrtReplicas, int countTlogReplicas,
+ int countPullReplicas, List<NodeBuilder> nodes) {
+ return initializeShardsReplicas(countShards, countNrtReplicas, countTlogReplicas, countPullReplicas, nodes, null);
+ }
+
+ /**
+ * Initializes shard and replica builders for the collection based on passed parameters. Replicas are assigned round
+ * robin to the nodes. The shard leader is the first NRT replica of each shard (or first TLOG is no NRT).
+ * Shard and replica configuration can be modified afterwards, the returned builder hierarchy is a convenient starting point.
+ * @param countShards number of shards to create
+ * @param countNrtReplicas number of NRT replicas per shard
+ * @param countTlogReplicas number of TLOG replicas per shard
+ * @param countPullReplicas number of PULL replicas per shard
+ * @param nodes list of nodes to place replicas on.
+ * @param initialSizeGBPerShard initial replica size (in GB) per shard
*/
public CollectionBuilder initializeShardsReplicas(int countShards, int countNrtReplicas, int countTlogReplicas,
- int countPullReplicas, List<NodeBuilder> nodes) {
+ int countPullReplicas, List<NodeBuilder> nodes,
+ List<Integer> initialSizeGBPerShard) {
Iterator<NodeBuilder> nodeIterator = nodes.iterator();
shardBuilders = new LinkedList<>();
+ if (initialSizeGBPerShard != null && initialSizeGBPerShard.size() != countShards) {
+ throw new RuntimeException("list of shard sizes must be the same length as the countShards!");
+ }
for (int shardNumber = 1; shardNumber <= countShards; shardNumber++) {
String shardName = buildShardName(shardNumber);
+ CollectionMetricsBuilder.ShardMetricsBuilder shardMetricsBuilder = new CollectionMetricsBuilder.ShardMetricsBuilder();
+
LinkedList<ReplicaBuilder> replicas = new LinkedList<>();
ReplicaBuilder leader = null;
+ CollectionMetricsBuilder.ReplicaMetricsBuilder leaderMetrics = null;
// Iterate on requested counts, NRT then TLOG then PULL. Leader chosen as first NRT (or first TLOG if no NRT)
List<Pair<Replica.ReplicaType, Integer>> replicaTypes = List.of(
@@ -270,15 +317,23 @@ public class Builders {
.setReplicaState(Replica.ReplicaState.ACTIVE).setReplicaNode(node);
replicas.add(replicaBuilder);
+ CollectionMetricsBuilder.ReplicaMetricsBuilder replicaMetricsBuilder = new CollectionMetricsBuilder.ReplicaMetricsBuilder();
+ shardMetricsBuilder.addReplicaMetrics(replicaName, replicaMetricsBuilder);
+ if (initialSizeGBPerShard != null) {
+ replicaMetricsBuilder.setSizeGB(initialSizeGBPerShard.get(shardNumber - 1));
+ }
if (leader == null && type != Replica.ReplicaType.PULL) {
leader = replicaBuilder;
+ leaderMetrics = replicaMetricsBuilder;
}
}
}
ShardBuilder shardBuilder = new ShardBuilder();
shardBuilder.setShardName(shardName).setReplicaBuilders(replicas).setLeader(leader);
+ shardMetricsBuilder.setLeaderMetrics(leaderMetrics);
shardBuilders.add(shardBuilder);
+ collectionMetricsBuilder.addShardMetrics(shardName, shardMetricsBuilder);
}
return this;
@@ -404,6 +459,8 @@ public class Builders {
private String nodeName = null;
private Integer coreCount = null;
private Long freeDiskGB = null;
+ private Long totalDiskGB = null;
+ private AttributeFetcher.DiskHardwareType diskType;
private Map<String, String> sysprops = null;
private Map<String, Double> metrics = null;
@@ -422,6 +479,11 @@ public class Builders {
return this;
}
+ public NodeBuilder setTotalDiskGB(Long totalDiskGB) {
+ this.totalDiskGB = totalDiskGB;
+ return this;
+ }
+
public NodeBuilder setSysprop(String key, String value) {
if (sysprops == null) {
sysprops = new HashMap<>();
@@ -431,6 +493,11 @@ public class Builders {
return this;
}
+ public NodeBuilder setDiskType(AttributeFetcher.DiskHardwareType diskType) {
+ this.diskType = diskType;
+ return this;
+ }
+
public NodeBuilder setMetric(AttributeFetcher.NodeMetricRegistry registry, String key, Double value) {
if (metrics == null) {
metrics = new HashMap<>();
@@ -448,6 +515,14 @@ public class Builders {
return freeDiskGB;
}
+ public Long getTotalDiskGB() {
+ return totalDiskGB;
+ }
+
+ public AttributeFetcher.DiskHardwareType getDiskType() {
+ return diskType;
+ }
+
public Map<String, String> getSysprops() {
return sysprops;
}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java
new file mode 100644
index 0000000..57cac4b
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/BuildersTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cluster.placement;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cluster.Cluster;
+import org.apache.solr.cluster.Node;
+import org.apache.solr.cluster.Shard;
+import org.apache.solr.cluster.SolrCollection;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.solr.cluster.placement.Builders.*;
+
+/**
+ *
+ */
+public class BuildersTest extends SolrTestCaseJ4 {
+
+ @Test
+ public void testClusterBuilder() throws Exception {
+ int NUM_NODES = 3;
+ int NUM_SHARDS = 2;
+ int NUM_NRT_REPLICAS = 2;
+ String collectionName = "test";
+ ClusterBuilder clusterBuilder = newClusterBuilder()
+ .initializeLiveNodes(NUM_NODES);
+ CollectionBuilder collectionBuilder = newCollectionBuilder(collectionName)
+ .initializeShardsReplicas(NUM_SHARDS,
+ NUM_NRT_REPLICAS,
+ NUM_NRT_REPLICAS + 1,
+ NUM_NRT_REPLICAS + 2,
+ clusterBuilder.getLiveNodeBuilders(),
+ List.of(10, 20));
+ clusterBuilder.addCollection(collectionBuilder);
+ Cluster cluster = clusterBuilder.build();
+ assertEquals("number of nodes", NUM_NODES, cluster.getLiveNodes().size());
+ SolrCollection collection = cluster.getCollection(collectionName);
+ assertNotNull("collection", collection);
+ assertEquals("shards", 2, collection.getShardNames().size());
+ for (String shardName : collection.getShardNames()) {
+ Shard shard = collection.getShard(shardName);
+ assertNotNull("shard leader", shard.getLeader());
+ int[] counts = new int[3];
+ shard.iterator().forEachRemaining(r -> {
+ switch (r.getType()) {
+ case NRT:
+ counts[0]++;
+ break;
+ case TLOG:
+ counts[1]++;
+ break;
+ case PULL:
+ counts[2]++;
+ }
+ });
+ assertEquals("numNrt", NUM_NRT_REPLICAS, counts[0]);
+ assertEquals("numTlog", NUM_NRT_REPLICAS + 1, counts[1]);
+ assertEquals("numPull", NUM_NRT_REPLICAS + 2, counts[2]);
+ }
+ // AttributeFetcher
+ AttributeFetcher attributeFetcher = clusterBuilder.buildAttributeFetcher();
+ attributeFetcher
+ .fetchFrom(cluster.getLiveNodes())
+ .requestNodeCoreCount()
+ .requestNodeDiskType()
+ .requestNodeFreeDisk()
+ .requestNodeTotalDisk()
+ .requestCollectionMetrics(collection, Set.of());
+ AttributeValues attributeValues = attributeFetcher.fetchAttributes();
+ for (Node node : cluster.getLiveNodes()) {
+ Optional<Integer> coreCount = attributeValues.getCoresCount(node);
+ assertTrue("coreCount present", coreCount.isPresent());
+ Optional<AttributeFetcher.DiskHardwareType> diskType = attributeValues.getDiskType(node);
+ assertTrue("diskType present", diskType.isPresent());
+ Optional<Long> diskOpt = attributeValues.getFreeDisk(node);
+ assertTrue("freeDisk", diskOpt.isPresent());
+ diskOpt = attributeValues.getTotalDisk(node);
+ assertTrue("totalDisk", diskOpt.isPresent());
+ }
+ Optional<CollectionMetrics> collectionMetricsOpt = attributeValues.getCollectionMetrics(collectionName);
+ assertTrue("collectionMetrics present", collectionMetricsOpt.isPresent());
+ CollectionMetrics collectionMetrics = collectionMetricsOpt.get();
+ for (String shardName : collection.getShardNames()) {
+ Optional<ShardMetrics> shardMetricsOpt = collectionMetrics.getShardMetrics(shardName);
+ assertTrue("shard metrics", shardMetricsOpt.isPresent());
+ ShardMetrics shardMetrics = shardMetricsOpt.get();
+ Optional<ReplicaMetrics> replicaMetricsOpt = shardMetrics.getLeaderMetrics();
+ assertTrue("leader metrics", replicaMetricsOpt.isPresent());
+ ReplicaMetrics leaderMetrics = replicaMetricsOpt.get();
+ if (shardName.endsWith("1")) {
+ assertEquals("size", 10, leaderMetrics.getReplicaSizeGB());
+ } else {
+ assertEquals("size", 20, leaderMetrics.getReplicaSizeGB());
+ }
+ Shard shard = collection.getShard(shardName);
+ shard.iterator().forEachRemaining(r -> {
+ Optional<ReplicaMetrics> metricsOpt = shardMetrics.getReplicaMetrics(r.getReplicaName());
+ assertTrue("replica metrics", metricsOpt.isPresent());
+ ReplicaMetrics metrics = metricsOpt.get();
+ if (shardName.endsWith("1")) {
+ assertEquals("size", 10, metrics.getReplicaSizeGB());
+ } else {
+ assertEquals("size", 20, metrics.getReplicaSizeGB());
+ }
+ });
+ }
+ }
+}