You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:09:38 UTC
[lucene] 06/33: SOLR-15004: Add a scalability test.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-15016
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 1d50edbc46000c9374ae13c92a8fbe07120178de
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Fri Nov 20 14:49:30 2020 +0100
SOLR-15004: Add a scalability test.
---
.../impl/AffinityPlacementFactoryTest.java | 91 ++++++++++++++++++++++
1 file changed, 91 insertions(+)
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
index 35bfcfc..48c7eb6 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.cluster.placement.impl;
import org.apache.solr.cluster.Cluster;
import org.apache.solr.cluster.Node;
+import org.apache.solr.cluster.Replica;
import org.apache.solr.cluster.Shard;
import org.apache.solr.cluster.SolrCollection;
import org.apache.solr.cluster.placement.*;
@@ -26,14 +27,22 @@ import org.apache.solr.cluster.placement.plugins.AffinityPlacementFactory;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* Unit test for {@link AffinityPlacementFactory}
*/
public class AffinityPlacementFactoryTest extends Assert {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static PlacementPlugin plugin;
@@ -100,4 +109,86 @@ public class AffinityPlacementFactoryTest extends Assert {
ReplicaPlacement rp = pp.getReplicaPlacements().iterator().next();
assertEquals(hasExistingCollection ? node2 : node1, rp.getNode());
}
+
+ @Test
+ public void testScalability() throws Exception {
+ log.info("==== numNodes ====");
+ runTestScalability(1000, 100, 40, 40, 20);
+ runTestScalability(2000, 100, 40, 40, 20);
+ runTestScalability(5000, 100, 40, 40, 20);
+ runTestScalability(10000, 100, 40, 40, 20);
+ runTestScalability(20000, 100, 40, 40, 20);
+ log.info("==== numShards ====");
+ runTestScalability(5000, 100, 40, 40, 20);
+ runTestScalability(5000, 200, 40, 40, 20);
+ runTestScalability(5000, 500, 40, 40, 20);
+ runTestScalability(5000, 1000, 40, 40, 20);
+ runTestScalability(5000, 2000, 40, 40, 20);
+ log.info("==== numReplicas ====");
+ runTestScalability(5000, 100, 100, 0, 0);
+ runTestScalability(5000, 100, 200, 0, 0);
+ runTestScalability(5000, 100, 500, 0, 0);
+ runTestScalability(5000, 100, 1000, 0, 0);
+ runTestScalability(5000, 100, 2000, 0, 0);
+ }
+
+ private void runTestScalability(int numNodes, int numShards,
+ int nrtReplicas, int tlogReplicas,
+ int pullReplicas) throws Exception {
+
+ int REPLICAS_PER_SHARD = nrtReplicas + tlogReplicas + pullReplicas;
+ int TOTAL_REPLICAS = numShards * REPLICAS_PER_SHARD;
+
+ String collectionName = "testCollection";
+
+ final Set<Node> liveNodes = new HashSet<>();
+ final Map<Node, Long> nodeToFreeDisk = new HashMap<>();
+ final Map<Node, Integer> nodeToCoreCount = new HashMap<>();
+ for (int i = 0; i < numNodes; i++) {
+ Node node = new ClusterAbstractionsForTest.NodeImpl("node_" + i);
+ liveNodes.add(node);
+ nodeToFreeDisk.put(node, Long.valueOf(numNodes));
+ nodeToCoreCount.put(node, 0);
+ }
+ ClusterAbstractionsForTest.SolrCollectionImpl solrCollection = new ClusterAbstractionsForTest.SolrCollectionImpl(collectionName, Map.of());
+ Map<String, Shard> shards = PluginTestHelper.createShardsAndReplicas(solrCollection, numShards, 0, Set.of());
+ solrCollection.setShards(shards);
+
+ Cluster cluster = new ClusterAbstractionsForTest.ClusterImpl(liveNodes, Map.of());
+ PlacementRequestImpl placementRequest = new PlacementRequestImpl(solrCollection, shards.keySet(), liveNodes,
+ nrtReplicas, tlogReplicas, pullReplicas);
+
+ AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk, Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
+ AttributeFetcher attributeFetcher = new AttributeFetcherForTest(attributeValues);
+ PlacementPlanFactory placementPlanFactory = new PlacementPlanFactoryImpl();
+
+ long start = System.nanoTime();
+ PlacementPlan pp = plugin.computePlacement(cluster, placementRequest, attributeFetcher, placementPlanFactory);
+ long end = System.nanoTime();
+ log.info("ComputePlacement: {} nodes, {} shards, {} total replicas, elapsed time {} ms.", numNodes, numShards, TOTAL_REPLICAS, TimeUnit.NANOSECONDS.toMillis(end - start));
+ assertEquals("incorrect number of calculated placements", TOTAL_REPLICAS,
+ pp.getReplicaPlacements().size());
+ // check that replicas are correctly placed
+ Map<Node, AtomicInteger> replicasPerNode = new HashMap<>();
+ Map<Node, Set<String>> shardsPerNode = new HashMap<>();
+ Map<String, AtomicInteger> replicasPerShard = new HashMap<>();
+ Map<Replica.ReplicaType, AtomicInteger> replicasByType = new HashMap<>();
+ for (ReplicaPlacement placement : pp.getReplicaPlacements()) {
+ replicasPerNode.computeIfAbsent(placement.getNode(), n -> new AtomicInteger()).incrementAndGet();
+ shardsPerNode.computeIfAbsent(placement.getNode(), n -> new HashSet<>()).add(placement.getShardName());
+ replicasByType.computeIfAbsent(placement.getReplicaType(), t -> new AtomicInteger()).incrementAndGet();
+ replicasPerShard.computeIfAbsent(placement.getShardName(), s -> new AtomicInteger()).incrementAndGet();
+ }
+ int perNode = TOTAL_REPLICAS > numNodes ? TOTAL_REPLICAS / numNodes : 1;
+ replicasPerNode.forEach((node, count) -> {
+ assertEquals(count.get(), perNode);
+ });
+ shardsPerNode.forEach((node, names) -> {
+ assertEquals(names.size(), perNode);
+ });
+
+ replicasPerShard.forEach((shard, count) -> {
+ assertEquals(count.get(), REPLICAS_PER_SHARD);
+ });
+ }
}