You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:09:38 UTC

[lucene] 06/33: SOLR-15004: Add a scalability test.

This is an automated email from the ASF dual-hosted git repository.

dweiss pushed a commit to branch jira/solr-15016
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit 1d50edbc46000c9374ae13c92a8fbe07120178de
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Fri Nov 20 14:49:30 2020 +0100

    SOLR-15004: Add a scalability test.
---
 .../impl/AffinityPlacementFactoryTest.java         | 91 ++++++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
index 35bfcfc..48c7eb6 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.cluster.placement.impl;
 
 import org.apache.solr.cluster.Cluster;
 import org.apache.solr.cluster.Node;
+import org.apache.solr.cluster.Replica;
 import org.apache.solr.cluster.Shard;
 import org.apache.solr.cluster.SolrCollection;
 import org.apache.solr.cluster.placement.*;
@@ -26,14 +27,22 @@ import org.apache.solr.cluster.placement.plugins.AffinityPlacementFactory;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * Unit test for {@link AffinityPlacementFactory}
  */
 public class AffinityPlacementFactoryTest extends Assert {
+    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
     private static PlacementPlugin plugin;
 
@@ -100,4 +109,86 @@ public class AffinityPlacementFactoryTest extends Assert {
         ReplicaPlacement rp = pp.getReplicaPlacements().iterator().next();
         assertEquals(hasExistingCollection ? node2 : node1, rp.getNode());
     }
+
+    @Test
+    public void testScalability() throws Exception {
+        log.info("==== numNodes ====");
+        runTestScalability(1000, 100, 40, 40, 20);
+        runTestScalability(2000, 100, 40, 40, 20);
+        runTestScalability(5000, 100, 40, 40, 20);
+        runTestScalability(10000, 100, 40, 40, 20);
+        runTestScalability(20000, 100, 40, 40, 20);
+        log.info("==== numShards ====");
+        runTestScalability(5000, 100, 40, 40, 20);
+        runTestScalability(5000, 200, 40, 40, 20);
+        runTestScalability(5000, 500, 40, 40, 20);
+        runTestScalability(5000, 1000, 40, 40, 20);
+        runTestScalability(5000, 2000, 40, 40, 20);
+        log.info("==== numReplicas ====");
+        runTestScalability(5000, 100, 100, 0, 0);
+        runTestScalability(5000, 100, 200, 0, 0);
+        runTestScalability(5000, 100, 500, 0, 0);
+        runTestScalability(5000, 100, 1000, 0, 0);
+        runTestScalability(5000, 100, 2000, 0, 0);
+    }
+
+    private void runTestScalability(int numNodes, int numShards,
+                                    int nrtReplicas, int tlogReplicas,
+                                    int pullReplicas) throws Exception {
+
+        int REPLICAS_PER_SHARD = nrtReplicas + tlogReplicas + pullReplicas;
+        int TOTAL_REPLICAS = numShards * REPLICAS_PER_SHARD;
+
+        String collectionName = "testCollection";
+
+        final Set<Node> liveNodes = new HashSet<>();
+        final Map<Node, Long> nodeToFreeDisk = new HashMap<>();
+        final Map<Node, Integer> nodeToCoreCount = new HashMap<>();
+        for (int i = 0; i < numNodes; i++) {
+            Node node = new ClusterAbstractionsForTest.NodeImpl("node_" + i);
+            liveNodes.add(node);
+            nodeToFreeDisk.put(node, Long.valueOf(numNodes));
+            nodeToCoreCount.put(node, 0);
+        }
+        ClusterAbstractionsForTest.SolrCollectionImpl solrCollection = new ClusterAbstractionsForTest.SolrCollectionImpl(collectionName, Map.of());
+        Map<String, Shard> shards = PluginTestHelper.createShardsAndReplicas(solrCollection, numShards, 0, Set.of());
+        solrCollection.setShards(shards);
+
+        Cluster cluster = new ClusterAbstractionsForTest.ClusterImpl(liveNodes, Map.of());
+        PlacementRequestImpl placementRequest = new PlacementRequestImpl(solrCollection, shards.keySet(), liveNodes,
+            nrtReplicas, tlogReplicas, pullReplicas);
+
+        AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk, Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
+        AttributeFetcher attributeFetcher = new AttributeFetcherForTest(attributeValues);
+        PlacementPlanFactory placementPlanFactory = new PlacementPlanFactoryImpl();
+
+        long start = System.nanoTime();
+        PlacementPlan pp = plugin.computePlacement(cluster, placementRequest, attributeFetcher, placementPlanFactory);
+        long end = System.nanoTime();
+        log.info("ComputePlacement: {} nodes, {} shards, {} total replicas, elapsed time {} ms.", numNodes, numShards, TOTAL_REPLICAS, TimeUnit.NANOSECONDS.toMillis(end - start));
+        assertEquals("incorrect number of calculated placements", TOTAL_REPLICAS,
+            pp.getReplicaPlacements().size());
+        // check that replicas are correctly placed
+        Map<Node, AtomicInteger> replicasPerNode = new HashMap<>();
+        Map<Node, Set<String>> shardsPerNode = new HashMap<>();
+        Map<String, AtomicInteger> replicasPerShard = new HashMap<>();
+        Map<Replica.ReplicaType, AtomicInteger> replicasByType = new HashMap<>();
+        for (ReplicaPlacement placement : pp.getReplicaPlacements()) {
+            replicasPerNode.computeIfAbsent(placement.getNode(), n -> new AtomicInteger()).incrementAndGet();
+            shardsPerNode.computeIfAbsent(placement.getNode(), n -> new HashSet<>()).add(placement.getShardName());
+            replicasByType.computeIfAbsent(placement.getReplicaType(), t -> new AtomicInteger()).incrementAndGet();
+            replicasPerShard.computeIfAbsent(placement.getShardName(), s -> new AtomicInteger()).incrementAndGet();
+        }
+        int perNode = TOTAL_REPLICAS > numNodes ? TOTAL_REPLICAS / numNodes : 1;
+        replicasPerNode.forEach((node, count) -> {
+            assertEquals(count.get(), perNode);
+        });
+        shardsPerNode.forEach((node, names) -> {
+            assertEquals(names.size(), perNode);
+        });
+
+        replicasPerShard.forEach((shard, count) -> {
+            assertEquals(count.get(), REPLICAS_PER_SHARD);
+        });
+    }
 }