You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by il...@apache.org on 2020/11/24 01:13:42 UTC
[lucene-solr] 01/02: Introduce a builder class for unit tests to
simplify creating cluster states and collections
This is an automated email from the ASF dual-hosted git repository.
ilan pushed a commit to branch jira/solr-15004
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit bd63ad16e5ab73ab2c794b2a1f01fe75fbf42a95
Author: Ilan Ginzburg <ig...@salesforce.com>
AuthorDate: Tue Nov 24 01:57:29 2020 +0100
Introduce a builder class for unit tests to simplify creating cluster states and collections
---
.../impl/AffinityPlacementFactoryTest.java | 52 +++-
.../solr/cluster/placement/impl/Builders.java | 287 +++++++++++++++++++++
.../placement/impl/ClusterAbstractionsForTest.java | 4 +-
3 files changed, 337 insertions(+), 6 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
index 80f30105..bf62c17 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
@@ -31,10 +31,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -63,6 +60,53 @@ public class AffinityPlacementFactoryTest extends Assert {
testBasicPlacementInternal(true);
}
+ @Test
+ public void testBasicPlacementNewCollection2() throws Exception {
+ testBasicInternal2(false);
+ }
+
+ @Test
+ public void testBasicPlacementExistingCollection2() throws Exception {
+ testBasicInternal2(true);
+ }
+
+ private void testBasicInternal2(boolean hasExistingCollection) throws Exception {
+ String collectionName = "testCollection";
+
+ Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeNodes(2);
+ LinkedList<Builders.NodeBuilder> nodeBuilders = clusterBuilder.getNodeBuilders();
+ nodeBuilders.get(0).setCoreCount(1).setFreeDiskGB(100L);
+ nodeBuilders.get(1).setCoreCount(10).setFreeDiskGB(100L);
+
+ Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder(collectionName);
+
+ if (hasExistingCollection) {
+ // Existing collection has replicas for its shards and is visible in the cluster state
+ collectionBuilder.initializeShardsReplicas(1, 1, 0, 0, nodeBuilders);
+ clusterBuilder.addCollection(collectionBuilder);
+ } else {
+ // New collection to create has the shards defined but no replicas and is not present in cluster state
+ collectionBuilder.initializeShardsReplicas(1, 0, 0, 0, List.of());
+ }
+
+ Cluster cluster = clusterBuilder.build();
+ AttributeFetcher attributeFetcher = clusterBuilder.buildAttributeFetcher();
+
+ SolrCollection solrCollection = collectionBuilder.build();
+ List<Node> liveNodes = clusterBuilder.buildLiveNodes();
+
+ // Place a new replica for the (only) existing shard of the collection
+ PlacementRequestImpl placementRequest = new PlacementRequestImpl(solrCollection,
+ Set.of(solrCollection.shards().iterator().next().getShardName()), new HashSet<>(liveNodes),
+ 1, 0, 0);
+
+ PlacementPlan pp = plugin.computePlacement(cluster, placementRequest, attributeFetcher, new PlacementPlanFactoryImpl());
+
+ assertEquals(1, pp.getReplicaPlacements().size());
+ ReplicaPlacement rp = pp.getReplicaPlacements().iterator().next();
+ assertEquals(hasExistingCollection ? liveNodes.get(1) : liveNodes.get(0), rp.getNode());
+ }
+
/**
* When this test places a replica for a new collection, it should pick the node with less cores.<p>
*
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/Builders.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/Builders.java
new file mode 100644
index 0000000..d954dd1
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/Builders.java
@@ -0,0 +1,287 @@
+package org.apache.solr.cluster.placement.impl;
+
+import org.apache.solr.cluster.*;
+import org.apache.solr.cluster.placement.AttributeFetcher;
+import org.apache.solr.cluster.placement.AttributeValues;
+import org.apache.solr.common.util.Pair;
+
+import java.util.*;
+
+/**
+ * Builder classes to make tests using different cluster and node configurations easier to write and to read.
+ */
+public class Builders {
+
+ public static ClusterBuilder newClusterBuilder() {
+ return new ClusterBuilder();
+ }
+
+ public static CollectionBuilder newCollectionBuilder(String collectionName) {
+ return new CollectionBuilder(collectionName);
+ }
+
+ static class ClusterBuilder {
+ private LinkedList<NodeBuilder> nodeBuilders = new LinkedList<>();
+ private LinkedList<CollectionBuilder> collectionBuilders = new LinkedList<>();
+
+ ClusterBuilder initializeNodes(int countNodes) {
+ nodeBuilders = new LinkedList<>();
+ for (int n = 0; n < countNodes; n++) {
+ nodeBuilders.add(new NodeBuilder().setNodeName("node" + n)); // Default name, can be changed
+ }
+ return this;
+ }
+
+ LinkedList<NodeBuilder> getNodeBuilders() {
+ return nodeBuilders;
+ }
+
+ ClusterBuilder addCollection(CollectionBuilder collectionBuilder) {
+ collectionBuilders.add(collectionBuilder);
+ return this;
+ }
+
+ Cluster build() {
+ // TODO if converting all tests to use builders change ClusterImpl ctor to use list of nodes
+ return new ClusterAbstractionsForTest.ClusterImpl(new HashSet<>(buildLiveNodes()), buildClusterCollections());
+ }
+
+ List<Node> buildLiveNodes() {
+ List<Node> liveNodes = new LinkedList<>();
+ for (NodeBuilder nodeBuilder : nodeBuilders) {
+ liveNodes.add(nodeBuilder.build());
+ }
+
+ return liveNodes;
+ }
+
+ Map<String, SolrCollection> buildClusterCollections() {
+ Map<String, SolrCollection> clusterCollections = new LinkedHashMap<>();
+ for (CollectionBuilder collectionBuilder : collectionBuilders) {
+ SolrCollection solrCollection = collectionBuilder.build();
+ clusterCollections.put(solrCollection.getName(), solrCollection);
+ }
+
+ return clusterCollections;
+ }
+
+ AttributeFetcher buildAttributeFetcher() {
+ Map<Node, Integer> nodeToCoreCount = new HashMap<>();
+ Map<Node, Long> nodeToFreeDisk = new HashMap<>();
+
+ // TODO And a few more missing and will be added...
+
+ // Slight redoing of work twice (building Node instances) but let's favor readability over tricks (I could think
+ // of many) to reuse the nodes computed in build() or build the AttributeFetcher at the same time.
+ for (NodeBuilder nodeBuilder : nodeBuilders) {
+ Node node = nodeBuilder.build();
+
+ if (nodeBuilder.getCoreCount() != null) {
+ nodeToCoreCount.put(node, nodeBuilder.getCoreCount());
+ }
+ if (nodeBuilder.getFreeDiskGB() != null) {
+ nodeToFreeDisk.put(node, nodeBuilder.getFreeDiskGB());
+ }
+ }
+
+ AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk, Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
+ return new AttributeFetcherForTest(attributeValues);
+ }
+ }
+
+ static class CollectionBuilder {
+ private final String collectionName;
+ private LinkedList<ShardBuilder> shardBuilders = new LinkedList<>();
+ private Map<String, String> customProperties = new HashMap<>();
+
+
+ private CollectionBuilder(String collectionName) {
+ this.collectionName = collectionName;
+ }
+
+ private CollectionBuilder addCustomProperty(String name, String value) {
+ customProperties.put(name, value);
+ return this;
+ }
+
+ /**
+ * Initializes shard and replica builders for the collection based on passed parameters. Replicas are assigned round
+ * robin to the nodes. The shard leader is the first NRT replica of each shard (or first TLOG is no NRT).
+ * Shard and replica configuration can be modified afterwards, the returned builder hierarchy is a convenient starting point.
+ */
+ CollectionBuilder initializeShardsReplicas(int countShards, int countNrtReplicas, int countTlogReplicas,
+ int countPullReplicas, List<NodeBuilder> nodes) {
+ Iterator<NodeBuilder> nodeIterator = nodes.iterator();
+
+ shardBuilders = new LinkedList<>();
+
+ for (int s = 0; s < countShards; s++) {
+ String shardName = collectionName + "_s" + s;
+
+ LinkedList<ReplicaBuilder> replicas = new LinkedList<>();
+ ReplicaBuilder leader = null;
+
+ // Iterate on requested counts, NRT then TLOG then PULL. Leader chosen as first NRT (or first TLOG if no NRT)
+ List<Pair<Replica.ReplicaType, Integer>> replicaTypes = List.of(
+ new Pair<>(Replica.ReplicaType.NRT, countNrtReplicas),
+ new Pair<>(Replica.ReplicaType.TLOG, countTlogReplicas),
+ new Pair<>(Replica.ReplicaType.PULL, countPullReplicas));
+
+ for (Pair<Replica.ReplicaType, Integer> tc : replicaTypes) {
+ Replica.ReplicaType type = tc.first();
+ int count = tc.second();
+ String replicaPrefix = shardName + "_" + type.name() + "_";
+ for (int r = 0; r < count; r++) {
+ String replicaName = replicaPrefix + r;
+ String coreName = replicaName + "_c";
+ if (!nodeIterator.hasNext()) {
+ nodeIterator = nodes.iterator();
+ }
+ // If the nodes set is empty, this call will fail
+ final NodeBuilder node = nodeIterator.next();
+
+ ReplicaBuilder replicaBuilder = new ReplicaBuilder();
+ replicaBuilder.setReplicaName(replicaName).setCoreName(coreName).setReplicaType(type)
+ .setReplicaState(Replica.ReplicaState.ACTIVE).setReplicaNode(node);
+ replicas.add(replicaBuilder);
+
+ if (leader == null && type != Replica.ReplicaType.PULL) {
+ leader = replicaBuilder;
+ }
+ }
+ }
+
+ ShardBuilder shardBuilder = new ShardBuilder();
+ shardBuilder.setShardName(shardName).setReplicaBuilders(replicas).setLeader(leader);
+ shardBuilders.add(shardBuilder);
+ }
+
+ return this;
+ }
+
+ SolrCollection build() {
+ ClusterAbstractionsForTest.SolrCollectionImpl solrCollection = new ClusterAbstractionsForTest.SolrCollectionImpl(collectionName, customProperties);
+
+ final LinkedHashMap<String, Shard> shards = new LinkedHashMap<>();
+
+ for (ShardBuilder shardBuilder : shardBuilders) {
+ Shard shard = shardBuilder.build(solrCollection);
+ shards.put(shard.getShardName(), shard);
+ }
+
+ solrCollection.setShards(shards);
+ return solrCollection;
+ }
+ }
+
+ static class ShardBuilder {
+ private String shardName;
+ private LinkedList<ReplicaBuilder> replicaBuilders = new LinkedList<>();
+ private ReplicaBuilder leaderReplicaBuilder;
+
+ ShardBuilder setShardName(String shardName) {
+ this.shardName = shardName;
+ return this;
+ }
+
+ ShardBuilder setReplicaBuilders(LinkedList<ReplicaBuilder> replicaBuilders) {
+ this.replicaBuilders = replicaBuilders;
+ return this;
+ }
+
+ ShardBuilder setLeader(ReplicaBuilder leaderReplicaBuilder) {
+ this.leaderReplicaBuilder = leaderReplicaBuilder;
+ return this;
+ }
+
+ Shard build(SolrCollection collection) {
+ ClusterAbstractionsForTest.ShardImpl shard = new ClusterAbstractionsForTest.ShardImpl(shardName, collection, Shard.ShardState.ACTIVE);
+
+ final LinkedHashMap<String, Replica> replicas = new LinkedHashMap<>();
+ Replica leader = null;
+
+ for (ReplicaBuilder replicaBuilder : replicaBuilders) {
+ Replica replica = replicaBuilder.build(shard);
+ replicas.put(replica.getReplicaName(), replica);
+
+ if (leaderReplicaBuilder == replicaBuilder) {
+ leader = replica;
+ }
+ }
+
+ shard.setReplicas(replicas, leader);
+ return shard;
+ }
+ }
+
+ static class ReplicaBuilder {
+ private String replicaName;
+ private String coreName;
+ private Replica.ReplicaType replicaType;
+ private Replica.ReplicaState replicaState;
+ private NodeBuilder replicaNode;
+
+ ReplicaBuilder setReplicaName(String replicaName) {
+ this.replicaName = replicaName;
+ return this;
+ }
+
+ ReplicaBuilder setCoreName(String coreName) {
+ this.coreName = coreName;
+ return this;
+ }
+
+ ReplicaBuilder setReplicaType(Replica.ReplicaType replicaType) {
+ this.replicaType = replicaType;
+ return this;
+ }
+
+ ReplicaBuilder setReplicaState(Replica.ReplicaState replicaState) {
+ this.replicaState = replicaState;
+ return this;
+ }
+
+ ReplicaBuilder setReplicaNode(NodeBuilder replicaNode) {
+ this.replicaNode = replicaNode;
+ return this;
+ }
+
+ Replica build(Shard shard) {
+ return new ClusterAbstractionsForTest.ReplicaImpl(replicaName, coreName, shard, replicaType, replicaState, replicaNode.build());
+ }
+ }
+
+ static class NodeBuilder {
+ private String nodeName = null;
+ private Integer coreCount = null;
+ private Long freeDiskGB = null;
+
+ NodeBuilder setNodeName(String nodeName) {
+ this.nodeName = nodeName;
+ return this;
+ }
+
+ NodeBuilder setCoreCount(Integer coreCount) {
+ this.coreCount = coreCount;
+ return this;
+ }
+
+ NodeBuilder setFreeDiskGB(Long freeDiskGB) {
+ this.freeDiskGB = freeDiskGB;
+ return this;
+ }
+
+ Integer getCoreCount() {
+ return coreCount;
+ }
+
+ Long getFreeDiskGB() {
+ return freeDiskGB;
+ }
+
+ Node build() {
+ // It is ok to build a new instance each time, that instance does the right thing with equals() and hashCode()
+ return new ClusterAbstractionsForTest.NodeImpl(nodeName);
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/ClusterAbstractionsForTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/ClusterAbstractionsForTest.java
index 188e3c3..69d3238 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/ClusterAbstractionsForTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/ClusterAbstractionsForTest.java
@@ -33,7 +33,7 @@ class ClusterAbstractionsForTest {
private final Set<Node> liveNodes;
private final Map<String, SolrCollection> collections;
- ClusterImpl(Set<Node> liveNodes, Map<String, SolrCollection> collections) throws IOException {
+ ClusterImpl(Set<Node> liveNodes, Map<String, SolrCollection> collections) {
this.liveNodes = liveNodes;
this.collections = collections;
}
@@ -88,7 +88,7 @@ class ClusterAbstractionsForTest {
/**
* This class ends up as a key in Maps in {@link org.apache.solr.cluster.placement.AttributeValues}.
* It is important to implement this method comparing node names given that new instances of {@link Node} are created
- * with names equal to existing instances (See {@link ReplicaImpl} constructor).
+ * with names equal to existing instances (See {@link Builders.NodeBuilder#build()}).
*/
public boolean equals(Object obj) {
if (obj == null) { return false; }