You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:09:42 UTC
[lucene] 10/33: Merge branch 'jira/solr-15004' of
https://github.com/apache/lucene-solr into jira/solr-15004 (and make
precommit happy)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-15016
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 0a112f5ffb7836a3a7ea944e89c43b21f84301b7
Merge: bd63ad1 9ca5920
Author: Ilan Ginzburg <ig...@salesforce.com>
AuthorDate: Tue Nov 24 02:12:16 2020 +0100
Merge branch 'jira/solr-15004' of https://github.com/apache/lucene-solr into jira/solr-15004 (and make precommit happy)
.../cluster/placement/impl/PlacementPlanImpl.java | 10 +++
.../placement/impl/ReplicaPlacementImpl.java | 5 ++
.../plugins/AffinityPlacementFactory.java | 42 ++++++-------
.../impl/AffinityPlacementFactoryTest.java | 73 ++++++++++++++++++----
.../placement/impl/ClusterAbstractionsForTest.java | 40 ++++++++++--
.../cluster/placement/impl/PluginTestHelper.java | 39 ++++++++++--
6 files changed, 164 insertions(+), 45 deletions(-)
diff --cc solr/core/src/java/org/apache/solr/cluster/placement/plugins/AffinityPlacementFactory.java
index 50ecb8d,9a00903..6982b8f
--- a/solr/core/src/java/org/apache/solr/cluster/placement/plugins/AffinityPlacementFactory.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/plugins/AffinityPlacementFactory.java
@@@ -118,6 -118,26 +118,26 @@@ public class AffinityPlacementFactory i
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
+ * <p>Name of the system property on a node indicating which (public cloud) Availability Zone that node is in. The value
+ * is any string, different strings denote different availability zones.
+ *
+ * <p>Nodes on which this system property is not defined are considered being in the same Availability Zone
+ * {@link #UNDEFINED_AVAILABILITY_ZONE} (hopefully the value of this constant is not the name of a real Availability Zone :).
+ */
+ public static final String AVAILABILITY_ZONE_SYSPROP = "availability_zone";
+
+ /**
+ * <p>Name of the system property on a node indicating the type of replicas allowed on that node.
+ * The value of that system property is a comma separated list or a single string of value names of
- * {@link Replica.ReplicaType} (case insensitive). If that property is not defined, that node is
++ * {@link org.apache.solr.cluster.Replica.ReplicaType} (case insensitive). If that property is not defined, that node is
+ * considered accepting all replica types (i.e. undefined is equivalent to {@code "NRT,Pull,tlog"}).
+ */
+ public static final String REPLICA_TYPE_SYSPROP = "replica_type";
+
+ /** This is the "AZ" name for nodes that do not define an AZ. Should not match a real AZ name (I think we're safe) */
+ public static final String UNDEFINED_AVAILABILITY_ZONE = "uNd3f1NeD";
+
+ /**
* Empty public constructor is used to instantiate this factory. Using a factory pattern to allow the factory to do one
* time costly operations if needed, and to only have to instantiate a default constructor class by name, rather than
* having to call a constructor with more parameters (if we were to instantiate the plugin class directly without going
diff --cc solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
index bf62c17,bcfc233..77f31bf
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/AffinityPlacementFactoryTest.java
@@@ -155,6 -113,54 +151,54 @@@ public class AffinityPlacementFactoryTe
}
@Test
+ public void testAvailabilityZones() throws Exception {
+ String collectionName = "testCollection";
+
+ int NUM_NODES = 6;
+ final Set<Node> liveNodes = new HashSet<>();
+ final Map<Node, Long> nodeToFreeDisk = new HashMap<>();
+ final Map<Node, Integer> nodeToCoreCount = new HashMap<>();
+ final Map<String, Map<Node, String>> zones = Map.of(AffinityPlacementFactory.AVAILABILITY_ZONE_SYSPROP, new HashMap<>());
+ final Map<Node, String> sysprops = zones.get(AffinityPlacementFactory.AVAILABILITY_ZONE_SYSPROP);
+ for (int i = 0; i < NUM_NODES; i++) {
+ Node node = new ClusterAbstractionsForTest.NodeImpl("node_" + i);
+ liveNodes.add(node);
+ nodeToFreeDisk.put(node, 100L);
+ nodeToCoreCount.put(node, 0);
+ if (i < NUM_NODES / 2) {
+ sysprops.put(node, "az1");
+ } else {
+ sysprops.put(node, "az2");
+ }
+ }
+
+ ClusterAbstractionsForTest.SolrCollectionImpl solrCollection = PluginTestHelper.createCollection(collectionName,
+ Map.of(), 2, 0, 0, 0, liveNodes);
+ ClusterAbstractionsForTest.ClusterImpl cluster = new ClusterAbstractionsForTest.ClusterImpl(liveNodes, Map.of());
+
+ PlacementRequestImpl placementRequest = new PlacementRequestImpl(solrCollection, solrCollection.getShardNames(), liveNodes, 2, 2, 2);
+
+ AttributeValues attributeValues = new AttributeValuesImpl(nodeToCoreCount, Map.of(), nodeToFreeDisk, Map.of(), Map.of(), Map.of(),
+ zones, Map.of());
+ AttributeFetcher attributeFetcher = new AttributeFetcherForTest(attributeValues);
+ PlacementPlanFactory placementPlanFactory = new PlacementPlanFactoryImpl();
+
+ PlacementPlan pp = plugin.computePlacement(cluster, placementRequest, attributeFetcher, placementPlanFactory);
+ // 2 shards, 5 replicas
+ assertEquals(12, pp.getReplicaPlacements().size());
+ List<ReplicaPlacement> placements = new ArrayList<>(pp.getReplicaPlacements());
+ Collections.sort(placements, Comparator
+ .comparing((ReplicaPlacement p) -> p.getNode().getName())
+ .thenComparing((ReplicaPlacement p) -> p.getShardName())
+ .thenComparing((ReplicaPlacement p) -> p.getReplicaType())
+ );
- log.info(placements.toString());
++ log.info(placements.toString()); //nowarn
+ // AZ -> shard -> replica count
+ //Map<String, Map<String, AtomicInteger>>
+ }
+
+ @Test
+ //@Ignore
public void testScalability() throws Exception {
log.info("==== numNodes ====");
runTestScalability(1000, 100, 40, 40, 20);
diff --cc solr/core/src/test/org/apache/solr/cluster/placement/impl/ClusterAbstractionsForTest.java
index 69d3238,02395d6..da82240
--- a/solr/core/src/test/org/apache/solr/cluster/placement/impl/ClusterAbstractionsForTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/impl/ClusterAbstractionsForTest.java
@@@ -20,7 -20,7 +20,6 @@@ package org.apache.solr.cluster.placeme
import org.apache.solr.cluster.*;
import javax.annotation.Nonnull;
--import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
@@@ -30,12 -30,16 +29,12 @@@
class ClusterAbstractionsForTest {
static class ClusterImpl implements Cluster {
- private final Set<Node> liveNodes;
- private final Map<String, SolrCollection> collections;
+ private final Set<Node> liveNodes = new HashSet<>();
+ private final Map<String, SolrCollection> collections = new HashMap<>();
- ClusterImpl() {
-
- }
-
- ClusterImpl(Set<Node> liveNodes, Map<String, SolrCollection> collections) throws IOException {
+ ClusterImpl(Set<Node> liveNodes, Map<String, SolrCollection> collections) {
- this.liveNodes = liveNodes;
- this.collections = collections;
+ this.liveNodes.addAll(liveNodes);
+ this.collections.putAll(collections);
}
@Override