You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:09:43 UTC
[lucene] 11/33: SOLR-15004: Extend the NodeBuilder,
move testAffinity to use the Builders.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-15016
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit cc52210997d5db75dc5a60540a6365b42d610824
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Tue Nov 24 13:43:11 2020 +0100
SOLR-15004: Extend the NodeBuilder, move testAffinity to use the Builders.
---
.../impl/AffinityPlacementFactoryTest.java | 43 ++++++++++----------
.../solr/cluster/placement/impl/Builders.java | 46 ++++++++++++++++++++--
2 files changed, 65 insertions(+), 24 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
index 77f31bf..6b64864 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
@@ -20,6 +20,7 @@ package org.apache.solr.cluster.placement.impl;
import org.apache.solr.cluster.Cluster;
import org.apache.solr.cluster.Node;
import org.apache.solr.cluster.Replica;
+import org.apache.solr.cluster.Shard;
import org.apache.solr.cluster.SolrCollection;
import org.apache.solr.cluster.placement.*;
import org.apache.solr.cluster.placement.plugins.AffinityPlacementFactory;
@@ -33,6 +34,8 @@ import java.lang.invoke.MethodHandles;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
/**
* Unit test for {@link AffinityPlacementFactory}
@@ -153,38 +156,36 @@ public class AffinityPlacementFactoryTest extends Assert {
@Test
public void testAvailabilityZones() throws Exception {
String collectionName = "testCollection";
-
int NUM_NODES = 6;
- final Set<Node> liveNodes = new HashSet<>();
- final Map<Node, Long> nodeToFreeDisk = new HashMap<>();
- final Map<Node, Integer> nodeToCoreCount = new HashMap<>();
- final Map<String, Map<Node, String>> zones = Map.of(AffinityPlacementFactory.AVAILABILITY_ZONE_SYSPROP, new HashMap<>());
- final Map<Node, String> sysprops = zones.get(AffinityPlacementFactory.AVAILABILITY_ZONE_SYSPROP);
+ Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeNodes(NUM_NODES);
for (int i = 0; i < NUM_NODES; i++) {
- Node node = new ClusterAbstractionsForTest.NodeImpl("node_" + i);
- liveNodes.add(node);
- nodeToFreeDisk.put(node, 100L);
- nodeToCoreCount.put(node, 0);
+ Builders.NodeBuilder nodeBuilder = clusterBuilder.getNodeBuilders().get(i);
+ nodeBuilder.setCoreCount(0);
+ nodeBuilder.setFreeDiskGB(100L);
if (i < NUM_NODES / 2) {
- sysprops.put(node, "az1");
+ nodeBuilder.setSysprop(AffinityPlacementFactory.AVAILABILITY_ZONE_SYSPROP, "az1");
} else {
- sysprops.put(node, "az2");
+ nodeBuilder.setSysprop(AffinityPlacementFactory.AVAILABILITY_ZONE_SYSPROP, "az2");
}
}
- ClusterAbstractionsForTest.SolrCollectionImpl solrCollection = PluginTestHelper.createCollection(collectionName,
- Map.of(), 2, 0, 0, 0, liveNodes);
- ClusterAbstractionsForTest.ClusterImpl cluster = new ClusterAbstractionsForTest.ClusterImpl(liveNodes, Map.of());
+ Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder(collectionName);
+ collectionBuilder.initializeShardsReplicas(2, 0, 0, 0, clusterBuilder.getNodeBuilders());
+ clusterBuilder.addCollection(collectionBuilder);
+
+ Cluster cluster = clusterBuilder.build();
- PlacementRequestImpl placementRequest = new PlacementRequestImpl(solrCollection, solrCollection.getShardNames(), liveNodes, 2, 2, 2);
+ SolrCollection solrCollection = cluster.getCollection(collectionName);
+
+ PlacementRequestImpl placementRequest = new PlacementRequestImpl(solrCollection,
+ StreamSupport.stream(solrCollection.shards().spliterator(), false)
+ .map(Shard::getShardName).collect(Collectors.toSet()),
+ cluster.getLiveNodes(), 2, 2, 2);
- AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk, Map.of(), Map.of(), Map.of(),
- zones, Map.of());
- AttributeFetcher attributeFetcher = new AttributeFetcherForTest(attributeValues);
PlacementPlanFactory placementPlanFactory = new PlacementPlanFactoryImpl();
- PlacementPlan pp = plugin.computePlacement(cluster, placementRequest, attributeFetcher, placementPlanFactory);
- // 2 shards, 5 replicas
+ PlacementPlan pp = plugin.computePlacement(cluster, placementRequest, clusterBuilder.buildAttributeFetcher(), placementPlanFactory);
+ // 2 shards, 6 replicas
assertEquals(12, pp.getReplicaPlacements().size());
List<ReplicaPlacement> placements = new ArrayList<>(pp.getReplicaPlacements());
Collections.sort(placements, Comparator
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/Builders.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/Builders.java
index d954dd1..996186f 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/Builders.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/Builders.java
@@ -68,6 +68,8 @@ public class Builders {
AttributeFetcher buildAttributeFetcher() {
Map<Node, Integer> nodeToCoreCount = new HashMap<>();
Map<Node, Long> nodeToFreeDisk = new HashMap<>();
+ Map<String, Map<Node, String>> sysprops = new HashMap<>();
+ Map<String, Map<Node, Double>> metrics = new HashMap<>();
// TODO And a few more missing and will be added...
@@ -82,9 +84,21 @@ public class Builders {
if (nodeBuilder.getFreeDiskGB() != null) {
nodeToFreeDisk.put(node, nodeBuilder.getFreeDiskGB());
}
+ if (nodeBuilder.getSysprops() != null) {
+ nodeBuilder.getSysprops().forEach((name, value) -> {
+ sysprops.computeIfAbsent(name, n -> new HashMap<>())
+ .put(node, value);
+ });
+ }
+ if (nodeBuilder.getMetrics() != null) {
+ nodeBuilder.getMetrics().forEach((name, value) -> {
+ metrics.computeIfAbsent(name, n -> new HashMap<>())
+ .put(node, value);
+ });
+ }
}
- AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk, Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
+ AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk, Map.of(), Map.of(), Map.of(), sysprops, metrics);
return new AttributeFetcherForTest(attributeValues);
}
}
@@ -116,7 +130,7 @@ public class Builders {
shardBuilders = new LinkedList<>();
for (int s = 0; s < countShards; s++) {
- String shardName = collectionName + "_s" + s;
+ String shardName = "shard" + (s + 1);
LinkedList<ReplicaBuilder> replicas = new LinkedList<>();
ReplicaBuilder leader = null;
@@ -130,7 +144,7 @@ public class Builders {
for (Pair<Replica.ReplicaType, Integer> tc : replicaTypes) {
Replica.ReplicaType type = tc.first();
int count = tc.second();
- String replicaPrefix = shardName + "_" + type.name() + "_";
+ String replicaPrefix = collectionName + "_" + shardName + "_replica_" + type.name().toLowerCase(Locale.ROOT).charAt(0);
for (int r = 0; r < count; r++) {
String replicaName = replicaPrefix + r;
String coreName = replicaName + "_c";
@@ -255,6 +269,8 @@ public class Builders {
private String nodeName = null;
private Integer coreCount = null;
private Long freeDiskGB = null;
+ private Map<String, String> sysprops = null;
+ private Map<String, Double> metrics = null;
NodeBuilder setNodeName(String nodeName) {
this.nodeName = nodeName;
@@ -271,6 +287,22 @@ public class Builders {
return this;
}
+ NodeBuilder setSysprop(String key, String value) {
+ if (sysprops == null) {
+ sysprops = new HashMap<>();
+ }
+ sysprops.put(key, value);
+ return this;
+ }
+
+ NodeBuilder setMetric(String key, Double value) {
+ if (metrics == null) {
+ metrics = new HashMap<>();
+ }
+ metrics.put(key, value);
+ return this;
+ }
+
Integer getCoreCount() {
return coreCount;
}
@@ -279,6 +311,14 @@ public class Builders {
return freeDiskGB;
}
+ Map<String, String> getSysprops() {
+ return sysprops;
+ }
+
+ Map<String, Double> getMetrics() {
+ return metrics;
+ }
+
Node build() {
// It is ok to build a new instance each time, that instance does the right thing with equals() and hashCode()
return new ClusterAbstractionsForTest.NodeImpl(nodeName);