You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2019/08/21 18:37:44 UTC
[lucene-solr] branch master updated: SOLR-13257: Support
deterministic replica routing
This is an automated email from the ASF dual-hosted git repository.
tflobbe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 8f4103d SOLR-13257: Support deterministic replica routing
8f4103d is described below
commit 8f4103dd4b7d572300aa585b9e25120fa97a25f3
Author: Tomas Fernandez Lobbe <tf...@apache.org>
AuthorDate: Wed Aug 21 11:31:12 2019 -0700
SOLR-13257: Support deterministic replica routing
Deterministic replica routing can help improve caching and allow a more consistent paging when sorting by score
This closes #677
---
solr/CHANGES.txt | 3 +
.../component/AffinityReplicaListTransformer.java | 118 +++++++++
.../AffinityReplicaListTransformerFactory.java | 91 +++++++
.../handler/component/HttpShardHandlerFactory.java | 283 ++++++++++++++++-----
.../component/ReplicaListTransformerFactory.java | 34 +++
.../component/TestHttpShardHandlerFactory.java | 103 +++++++-
solr/solr-ref-guide/src/distributed-requests.adoc | 12 +
solr/solr-ref-guide/src/format-of-solr-xml.adoc | 14 +
.../org/apache/solr/common/params/ShardParams.java | 15 ++
9 files changed, 614 insertions(+), 59 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 77043a8..c2029db 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -114,6 +114,9 @@ New Features
* SOLR-13650: Solr now can define and add "packages" with plugins. Each plugin can choose to
load from one of those packages & updating packages can reload those plugins independently (noble)
+* SOLR-13257: Support deterministic replica routing preferences for better cache usage (Michael Gibney
+ via Christine Poerschke, Tomás Fernández Löbbe)
+
Improvements
----------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformer.java b/solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformer.java
new file mode 100644
index 0000000..94334f2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformer.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Hash;
+import org.apache.solr.request.SolrQueryRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Allows better caching by establishing deterministic evenly-distributed replica routing preferences according to
+ * either explicitly configured hash routing parameter, or the hash of a query parameter (configurable, usually related
+ * to the main query).
+ */
+class AffinityReplicaListTransformer implements ReplicaListTransformer {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final int routingDividend;
+
+ public AffinityReplicaListTransformer(String hashVal) {
+ this.routingDividend = Math.abs(Hash.lookup3ycs(hashVal, 0, hashVal.length(), 0));
+ }
+
+ public AffinityReplicaListTransformer(int routingDividend) {
+ this.routingDividend = routingDividend;
+ }
+
+ /**
+ *
+ * @param dividendParam int param to be used directly for mod-based routing
+ * @param hashParam String param to be hashed into an int for mod-based routing
+ * @param req the request from which param values will be drawn
+ * @return null if specified routing vals are not able to be parsed properly
+ */
+ public static ReplicaListTransformer getInstance(String dividendParam, String hashParam, SolrQueryRequest req) {
+ SolrParams params = req.getOriginalParams();
+ Integer dividendVal;
+ if (dividendParam != null && (dividendVal = params.getInt(dividendParam)) != null) {
+ return new AffinityReplicaListTransformer(dividendVal);
+ }
+ String hashVal;
+ if (hashParam != null && (hashVal = params.get(hashParam)) != null && !hashVal.isEmpty()) {
+ return new AffinityReplicaListTransformer(hashVal);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void transform(List<?> choices) {
+ int size = choices.size();
+ if (size > 1) {
+ int i = 0;
+ SortableChoice[] sortableChoices = new SortableChoice[size];
+ for (Object o : choices) {
+ sortableChoices[i++] = new SortableChoice(o);
+ }
+ Arrays.sort(sortableChoices, SORTABLE_CHOICE_COMPARATOR);
+ ListIterator iter = choices.listIterator();
+ i = routingDividend % size;
+ final int limit = i + size;
+ do {
+ iter.next();
+ iter.set(sortableChoices[i % size].choice);
+ } while (++i < limit);
+ }
+ }
+
+ private static final class SortableChoice {
+
+ private final Object choice;
+ private final String sortableCoreLabel;
+
+ private SortableChoice(Object choice) {
+ this.choice = choice;
+ if (choice instanceof Replica) {
+ this.sortableCoreLabel = ((Replica)choice).getCoreUrl();
+ } else if (choice instanceof String) {
+ this.sortableCoreLabel = (String)choice;
+ } else {
+ throw new IllegalArgumentException("can't handle type " + choice.getClass());
+ }
+ }
+
+ }
+
+ private static final Comparator<SortableChoice> SORTABLE_CHOICE_COMPARATOR = new Comparator<SortableChoice>() {
+
+ @Override
+ public int compare(SortableChoice o1, SortableChoice o2) {
+ return o1.sortableCoreLabel.compareTo(o2.sortableCoreLabel);
+ }
+ };
+
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformerFactory.java
new file mode 100644
index 0000000..1e97c25
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/AffinityReplicaListTransformerFactory.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+
+/**
+ * Factory for constructing an {@link AffinityReplicaListTransformer} that reorders replica routing
+ * preferences deterministically, based on request parameters.
+ *
+ * Default names of params that contain the values by which routing is determined may be configured
+ * at the time of {@link AffinityReplicaListTransformerFactory} construction, and may be
+ * overridden by the config spec passed to {@link #getInstance(String, SolrQueryRequest, ReplicaListTransformerFactory)}
+ *
+ * If no defaultHashParam name is specified at time of factory construction, the routing dividend will
+ * be derived by hashing the {@link String} value of the {@link CommonParams#Q} param.
+ */
+class AffinityReplicaListTransformerFactory implements ReplicaListTransformerFactory {
+ private final String defaultDividendParam;
+ private final String defaultHashParam;
+
+ public AffinityReplicaListTransformerFactory() {
+ this.defaultDividendParam = null;
+ this.defaultHashParam = CommonParams.Q;
+ }
+
+ public AffinityReplicaListTransformerFactory(String defaultDividendParam, String defaultHashParam) {
+ this.defaultDividendParam = defaultDividendParam;
+ this.defaultHashParam = defaultHashParam;
+ }
+
+ public AffinityReplicaListTransformerFactory(NamedList<?> c) {
+ this((String)c.get(ShardParams.ROUTING_DIVIDEND), translateHashParam((String)c.get(ShardParams.ROUTING_HASH)));
+ }
+
+ /**
+ * Null arg indicates no configuration, which should be translated to the default value {@link CommonParams#Q}.
+ * Empty String is translated to null, allowing users to explicitly disable hash-based stable routing.
+ *
+ * @param hashParam configured hash param (null indicates unconfigured).
+ * @return translated value to be used as default hash param in RLT.
+ */
+ private static String translateHashParam(String hashParam) {
+ if (hashParam == null) {
+ return CommonParams.Q;
+ } else if (hashParam.isEmpty()) {
+ return null;
+ } else {
+ return hashParam;
+ }
+ }
+
+ @Override
+ public ReplicaListTransformer getInstance(String configSpec, SolrQueryRequest request, ReplicaListTransformerFactory fallback) {
+ ReplicaListTransformer rlt;
+ if (configSpec == null) {
+ rlt = AffinityReplicaListTransformer.getInstance(defaultDividendParam, defaultHashParam, request);
+ } else {
+ String[] parts = configSpec.split(":", 2);
+ switch (parts[0]) {
+ case ShardParams.ROUTING_DIVIDEND:
+ rlt = AffinityReplicaListTransformer.getInstance(parts.length == 1 ? defaultDividendParam : parts[1], defaultHashParam, request);
+ break;
+ case ShardParams.ROUTING_HASH:
+ rlt = AffinityReplicaListTransformer.getInstance(null, parts.length == 1 ? defaultHashParam : parts[1], request);
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid routing spec: \"" + configSpec + '"');
+ }
+ }
+ return rlt != null ? rlt : fallback.getInstance(null, request, null);
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
index cf16fa2..844acf3 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
@@ -27,8 +27,10 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
@@ -67,6 +69,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.URLUtil;
import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
@@ -197,6 +200,61 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
return Boolean.getBoolean(INIT_SOLR_DISABLE_SHARDS_WHITELIST);
}
+ private static NamedList<?> getNamedList(Object val) {
+ if (val instanceof NamedList) {
+ return (NamedList<?>)val;
+ } else {
+ throw new IllegalArgumentException("Invalid config for replicaRouting; expected NamedList, but got " + val);
+ }
+ }
+
+ private static String checkDefaultReplicaListTransformer(NamedList<?> c, String setTo, String extantDefaultRouting) {
+ if (!Boolean.TRUE.equals(c.getBooleanArg("default"))) {
+ return null;
+ } else {
+ if (extantDefaultRouting == null) {
+ return setTo;
+ } else {
+ throw new IllegalArgumentException("more than one routing scheme marked as default");
+ }
+ }
+ }
+
+ private void initReplicaListTransformers(NamedList routingConfig) {
+ String defaultRouting = null;
+ if (routingConfig != null && routingConfig.size() > 0) {
+ Iterator<Entry<String,?>> iter = routingConfig.iterator();
+ do {
+ Entry<String, ?> e = iter.next();
+ String key = e.getKey();
+ switch (key) {
+ case ShardParams.REPLICA_RANDOM:
+ // Only positive assertion of default status (i.e., default=true) is supported.
+ // "random" is currently the implicit default, so explicitly configuring
+ // "random" as default would not currently be useful, but if the implicit default
+ // changes in the future, checkDefault could be relevant here.
+ defaultRouting = checkDefaultReplicaListTransformer(getNamedList(e.getValue()), key, defaultRouting);
+ break;
+ case ShardParams.REPLICA_STABLE:
+ NamedList<?> c = getNamedList(e.getValue());
+ defaultRouting = checkDefaultReplicaListTransformer(c, key, defaultRouting);
+ this.stableRltFactory = new AffinityReplicaListTransformerFactory(c);
+ break;
+ default:
+ throw new IllegalArgumentException("invalid replica routing spec name: " + key);
+ }
+ } while (iter.hasNext());
+ }
+ if (this.stableRltFactory == null) {
+ this.stableRltFactory = new AffinityReplicaListTransformerFactory();
+ }
+ if (ShardParams.REPLICA_STABLE.equals(defaultRouting)) {
+ this.defaultRltFactory = this.stableRltFactory;
+ } else {
+ this.defaultRltFactory = this.randomRltFactory;
+ }
+ }
+
@Override
public void init(PluginInfo info) {
StringBuilder sb = new StringBuilder();
@@ -231,8 +289,6 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
this.whitelistHostChecker = new WhitelistHostChecker(args == null? null: (String) args.get(INIT_SHARDS_WHITELIST), !getDisableShardsWhitelist());
log.info("Host whitelist initialized: {}", this.whitelistHostChecker);
- log.debug("created with {}",sb);
-
// magic sysprop to make tests reproducible: set by SolrTestCaseJ4.
String v = System.getProperty("tests.shardhandler.randomSeed");
if (v != null) {
@@ -265,6 +321,10 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
.maxConnectionsPerHost(maxConnectionsPerHost).build();
this.defaultClient.addListenerFactory(this.httpListenerFactory);
this.loadbalancer = new LBHttp2SolrClient(defaultClient);
+
+ initReplicaListTransformers(getParameter(args, "replicaRouting", null, sb));
+
+ log.debug("created with {}",sb);
}
@Override
@@ -344,18 +404,26 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
* E.g. If all nodes prefer local cores then a bad/heavily-loaded node will receive less requests from
* healthy nodes. This will help prevent a distributed deadlock or timeouts in all the healthy nodes due
* to one bad node.
+ *
+ * Optional final preferenceRule is *not* used for pairwise sorting, but instead defines how "equivalent"
+ * replicas will be ordered (the base ordering). Defaults to "random"; may specify "stable".
*/
static class NodePreferenceRulesComparator implements Comparator<Object> {
private final SolrQueryRequest request;
private final NodesSysPropsCacher sysPropsCache;
private final String nodeName;
- private List<PreferenceRule> preferenceRules;
+ private final List<PreferenceRule> sortRules;
+ private final List<PreferenceRule> preferenceRules;
private String localHostAddress = null;
+ private final ReplicaListTransformer baseReplicaListTransformer;
- public NodePreferenceRulesComparator(final List<PreferenceRule> sortRules, final SolrQueryRequest request) {
+ public NodePreferenceRulesComparator(final List<PreferenceRule> preferenceRules, final SolrQueryRequest request,
+ final ReplicaListTransformerFactory defaultRltFactory, final ReplicaListTransformerFactory randomRltFactory,
+ final ReplicaListTransformerFactory stableRltFactory) {
this.request = request;
- if (request != null && request.getCore().getCoreContainer().getZkController() != null) {
+ final SolrCore core; // explicit check for null core (temporary?, for tests)
+ if (request != null && (core = request.getCore()) != null && core.getCoreContainer().getZkController() != null) {
ZkController zkController = request.getCore().getCoreContainer().getZkController();
sysPropsCache = zkController.getSysPropsCacher();
nodeName = zkController.getNodeName();
@@ -363,36 +431,77 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
sysPropsCache = null;
nodeName = null;
}
- this.preferenceRules = sortRules;
- }
-
- @Override
- public int compare(Object left, Object right) {
- for (PreferenceRule preferenceRule: this.preferenceRules) {
- final boolean lhs;
- final boolean rhs;
- switch (preferenceRule.name) {
- case ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE:
- lhs = hasReplicaType(left, preferenceRule.value);
- rhs = hasReplicaType(right, preferenceRule.value);
- break;
- case ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION:
- lhs = hasCoreUrlPrefix(left, preferenceRule.value);
- rhs = hasCoreUrlPrefix(right, preferenceRule.value);
+ this.preferenceRules = preferenceRules;
+ final int maxIdx = preferenceRules.size() - 1;
+ final PreferenceRule lastRule = preferenceRules.get(maxIdx);
+ if (!ShardParams.SHARDS_PREFERENCE_REPLICA_BASE.equals(lastRule.name)) {
+ this.sortRules = preferenceRules;
+ this.baseReplicaListTransformer = defaultRltFactory.getInstance(null, request, randomRltFactory);
+ } else {
+ if (maxIdx == 0) {
+ this.sortRules = null;
+ } else {
+ this.sortRules = preferenceRules.subList(0, maxIdx);
+ }
+ String[] parts = lastRule.value.split(":", 2);
+ switch (parts[0]) {
+ case ShardParams.REPLICA_RANDOM:
+ this.baseReplicaListTransformer = randomRltFactory.getInstance(parts.length == 1 ? null : parts[1], request, null);
break;
- case ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP:
- if (sysPropsCache == null) {
- throw new IllegalArgumentException("Unable to get the NodesSysPropsCacher" +
- " on sorting replicas by preference:"+ preferenceRule.value);
- }
- lhs = hasSameMetric(left, preferenceRule.value);
- rhs = hasSameMetric(right, preferenceRule.value);
+ case ShardParams.REPLICA_STABLE:
+ this.baseReplicaListTransformer = stableRltFactory.getInstance(parts.length == 1 ? null : parts[1], request, randomRltFactory);
break;
default:
- throw new IllegalArgumentException("Invalid " + ShardParams.SHARDS_PREFERENCE + " type: " + preferenceRule.name);
+ throw new IllegalArgumentException("Invalid base replica order spec");
}
- if (lhs != rhs) {
- return lhs ? -1 : +1;
+ }
+ }
+ private static final ReplicaListTransformer NOOP_RLT = (List<?> choices) -> { /* noop */ };
+ private static final ReplicaListTransformerFactory NOOP_RLTF = (String configSpec, SolrQueryRequest request,
+ ReplicaListTransformerFactory fallback) -> NOOP_RLT;
+ /**
+ * For compatibility with tests, which expect this constructor to have no effect on the *base* order.
+ */
+ NodePreferenceRulesComparator(final List<PreferenceRule> sortRules, final SolrQueryRequest request) {
+ this(sortRules, request, NOOP_RLTF, null, null);
+ }
+
+ public ReplicaListTransformer getBaseReplicaListTransformer() {
+ return baseReplicaListTransformer;
+ }
+
+ @Override
+ public int compare(Object left, Object right) {
+ if (this.sortRules != null) {
+ for (PreferenceRule preferenceRule: this.sortRules) {
+ final boolean lhs;
+ final boolean rhs;
+ switch (preferenceRule.name) {
+ case ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE:
+ lhs = hasReplicaType(left, preferenceRule.value);
+ rhs = hasReplicaType(right, preferenceRule.value);
+ break;
+ case ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION:
+ lhs = hasCoreUrlPrefix(left, preferenceRule.value);
+ rhs = hasCoreUrlPrefix(right, preferenceRule.value);
+ break;
+ case ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP:
+ if (sysPropsCache == null) {
+ throw new IllegalArgumentException("Unable to get the NodesSysPropsCacher" +
+ " on sorting replicas by preference:"+ preferenceRule.value);
+ }
+ lhs = hasSameMetric(left, preferenceRule.value);
+ rhs = hasSameMetric(right, preferenceRule.value);
+ break;
+ case ShardParams.SHARDS_PREFERENCE_REPLICA_BASE:
+ throw new IllegalArgumentException("only one base replica order may be specified in "
+ + ShardParams.SHARDS_PREFERENCE + ", and it must be specified last");
+ default:
+ throw new IllegalArgumentException("Invalid " + ShardParams.SHARDS_PREFERENCE + " type: " + preferenceRule.name);
+ }
+ if (lhs != rhs) {
+ return lhs ? -1 : +1;
+ }
}
}
return 0;
@@ -449,9 +558,83 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
}
}
+ private final ReplicaListTransformerFactory randomRltFactory = (String configSpec, SolrQueryRequest request,
+ ReplicaListTransformerFactory fallback) -> shufflingReplicaListTransformer;
+ private ReplicaListTransformerFactory stableRltFactory;
+ private ReplicaListTransformerFactory defaultRltFactory;
+
+ /**
+ * Private class responsible for applying pairwise sort based on inherent replica attributes,
+ * and subsequently reordering any equivalent replica sets according to behavior specified
+ * by the baseReplicaListTransformer.
+ */
+ private static final class TopLevelReplicaListTransformer implements ReplicaListTransformer {
+
+ private final NodePreferenceRulesComparator replicaComp;
+ private final ReplicaListTransformer baseReplicaListTransformer;
+
+ public TopLevelReplicaListTransformer(NodePreferenceRulesComparator replicaComp, ReplicaListTransformer baseReplicaListTransformer) {
+ this.replicaComp = replicaComp;
+ this.baseReplicaListTransformer = baseReplicaListTransformer;
+ }
+
+ @Override
+ public void transform(List<?> choices) {
+ if (choices.size() > 1) {
+ if (log.isDebugEnabled()) {
+ log.debug("Applying the following sorting preferences to replicas: {}",
+ Arrays.toString(replicaComp.preferenceRules.toArray()));
+ }
+
+ // First, sort according to comparator rules.
+ try {
+ choices.sort(replicaComp);
+ } catch (IllegalArgumentException iae) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ iae.getMessage()
+ );
+ }
+
+ // Next determine all boundaries between replicas ranked as "equivalent" by the comparator
+ Iterator<?> iter = choices.iterator();
+ Object prev = iter.next();
+ Object current;
+ int idx = 1;
+ int boundaryCount = 0;
+ int[] boundaries = new int[choices.size() - 1];
+ do {
+ current = iter.next();
+ if (replicaComp.compare(prev, current) != 0) {
+ boundaries[boundaryCount++] = idx;
+ }
+ prev = current;
+ idx++;
+ } while (iter.hasNext());
+
+ // Finally inspect boundaries to apply base transformation, where necessary (separate phase to avoid ConcurrentModificationException)
+ int startIdx = 0;
+ int endIdx;
+ for (int i = 0; i < boundaryCount; i++) {
+ endIdx = boundaries[i];
+ if (endIdx - startIdx > 1) {
+ baseReplicaListTransformer.transform(choices.subList(startIdx, endIdx));
+ }
+ startIdx = endIdx;
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Applied sorting preferences to replica list: {}",
+ Arrays.toString(choices.toArray()));
+ }
+ }
+ }
+ }
+
protected ReplicaListTransformer getReplicaListTransformer(final SolrQueryRequest req) {
final SolrParams params = req.getParams();
- ZkController zkController = req.getCore().getCoreContainer().getZkController();
+ final SolrCore core = req.getCore(); // explicit check for null core (temporary?, for tests)
+ ZkController zkController = core == null ? null : core.getCoreContainer().getZkController();
String defaultShardPreference = "";
if (zkController != null) {
defaultShardPreference = zkController.getZkStateReader().getClusterProperties()
@@ -476,34 +659,18 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
preferenceRules.add(new PreferenceRule(ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION, ShardParams.REPLICA_LOCAL));
}
- return new ShufflingReplicaListTransformer(r) {
- @Override
- public void transform(List<?> choices)
- {
- if (choices.size() > 1) {
- super.transform(choices);
- if (log.isDebugEnabled()) {
- log.debug("Applying the following sorting preferences to replicas: {}",
- Arrays.toString(preferenceRules.toArray()));
- }
- try {
- choices.sort(new NodePreferenceRulesComparator(preferenceRules, req));
- } catch (IllegalArgumentException iae) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST,
- iae.getMessage()
- );
- }
- if (log.isDebugEnabled()) {
- log.debug("Applied sorting preferences to replica list: {}",
- Arrays.toString(choices.toArray()));
- }
- }
- }
- };
+ NodePreferenceRulesComparator replicaComp = new NodePreferenceRulesComparator(preferenceRules, req,
+ defaultRltFactory, randomRltFactory, stableRltFactory);
+ ReplicaListTransformer baseReplicaListTransformer = replicaComp.getBaseReplicaListTransformer();
+ if (replicaComp.sortRules == null) {
+ // only applying base transformation
+ return baseReplicaListTransformer;
+ } else {
+ return new TopLevelReplicaListTransformer(replicaComp, baseReplicaListTransformer);
+ }
}
- return shufflingReplicaListTransformer;
+ return defaultRltFactory.getInstance(null, req, randomRltFactory);
}
/**
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ReplicaListTransformerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/ReplicaListTransformerFactory.java
new file mode 100644
index 0000000..af09a69
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/ReplicaListTransformerFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.request.SolrQueryRequest;
+
+public interface ReplicaListTransformerFactory {
+
+ /**
+ *
+ * @param configSpec spec for dynamic configuration of ReplicaListTransformer
+ * @param request the request for which the ReplicaListTransformer is being generated
+ * @param fallback used to generate fallback value; the getInstance() method of the specified fallback must not
+ * return null; The fallback value itself may be null if this implementation is known to never return null (i.e., if
+ * fallback will never be needed)
+ * @return ReplicaListTransformer to be used for routing this request
+ */
+ ReplicaListTransformer getInstance(String configSpec, SolrQueryRequest request, ReplicaListTransformerFactory fallback);
+
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
index 89d5ba1..f995a7f 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
@@ -20,6 +20,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -34,8 +35,12 @@ import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.HttpShardHandlerFactory.WhitelistHostChecker;
+import org.apache.solr.request.SolrQueryRequestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -117,7 +122,97 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 {
}
@SuppressWarnings("unchecked")
- public void testNodePreferenceRulesComparator() throws Exception {
+ public void testNodePreferenceRulesBase() throws Exception {
+ SolrCore testCore = null;
+ HttpShardHandlerFactory fac = new HttpShardHandlerFactory();
+ fac.init(new PluginInfo(null, Collections.EMPTY_MAP));
+ SolrQueryRequestBase req;
+ NamedList<String> params = new NamedList<>();
+ List<Replica> replicas = getBasicReplicaList();
+
+ String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
+
+ params.add("routingPreference", "0");
+ params.add(ShardParams.SHARDS_PREFERENCE, rulesParam);
+
+ req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
+ ReplicaListTransformer rlt = fac.getReplicaListTransformer(req);
+ rlt.transform(replicas);
+ assertEquals("node1", replicas.get(0).getNodeName());
+ assertEquals("node2", replicas.get(1).getNodeName());
+ assertEquals("node3", replicas.get(2).getNodeName());
+ req.close();
+
+ params.setVal(0, "1");
+ req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
+ rlt = fac.getReplicaListTransformer(req);
+ rlt.transform(replicas);
+ assertEquals("node2", replicas.get(0).getNodeName());
+ assertEquals("node3", replicas.get(1).getNodeName());
+ assertEquals("node1", replicas.get(2).getNodeName());
+ req.close();
+
+ params.setVal(0, "2");
+ req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
+ rlt = fac.getReplicaListTransformer(req);
+ rlt.transform(replicas);
+ assertEquals("node3", replicas.get(0).getNodeName());
+ assertEquals("node1", replicas.get(1).getNodeName());
+ assertEquals("node2", replicas.get(2).getNodeName());
+ req.close();
+
+ params.setVal(0, "3");
+ req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
+ rlt = fac.getReplicaListTransformer(req);
+ rlt.transform(replicas);
+ assertEquals("node1", replicas.get(0).getNodeName());
+ assertEquals("node2", replicas.get(1).getNodeName());
+ assertEquals("node3", replicas.get(2).getNodeName());
+ req.close();
+
+ // Add a replica so that sorting by replicaType:TLOG can cause a tie
+ replicas.add(
+ new Replica(
+ "node4",
+ map(
+ ZkStateReader.BASE_URL_PROP, "http://host2_2:8983/solr",
+ ZkStateReader.NODE_NAME_PROP, "node4",
+ ZkStateReader.CORE_NAME_PROP, "collection1",
+ ZkStateReader.REPLICA_TYPE, "TLOG"
+ )
+ )
+ );
+
+ // replicaType and replicaBase combined rule param
+ rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
+ ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
+ ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";
+
+ params.setVal(0, "0");
+ params.setVal(1, rulesParam);
+ req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
+ rlt = fac.getReplicaListTransformer(req);
+ rlt.transform(replicas);
+ assertEquals("node1", replicas.get(0).getNodeName());
+ assertEquals("node2", replicas.get(1).getNodeName());
+ assertEquals("node4", replicas.get(2).getNodeName());
+ assertEquals("node3", replicas.get(3).getNodeName());
+ req.close();
+
+ params.setVal(0, "1");
+ req = new SolrQueryRequestBase(testCore, params.toSolrParams()) {};
+ rlt = fac.getReplicaListTransformer(req);
+ rlt.transform(replicas);
+ assertEquals("node1", replicas.get(0).getNodeName());
+ assertEquals("node4", replicas.get(1).getNodeName());
+ assertEquals("node2", replicas.get(2).getNodeName());
+ assertEquals("node3", replicas.get(3).getNodeName());
+ req.close();
+ fac.close();
+ }
+
+ @SuppressWarnings("unchecked")
+ private static List<Replica> getBasicReplicaList() {
List<Replica> replicas = new ArrayList<Replica>();
replicas.add(
new Replica(
@@ -152,6 +247,12 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 {
)
)
);
+ return replicas;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void testNodePreferenceRulesComparator() throws Exception {
+ List<Replica> replicas = getBasicReplicaList();
// Simple replica type rule
List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
diff --git a/solr/solr-ref-guide/src/distributed-requests.adoc b/solr/solr-ref-guide/src/distributed-requests.adoc
index 3422540..259aa8f 100644
--- a/solr/solr-ref-guide/src/distributed-requests.adoc
+++ b/solr/solr-ref-guide/src/distributed-requests.adoc
@@ -180,8 +180,20 @@ In other words, this feature is mostly useful for optimizing queries directed to
+
Also, this option should only be used if you are load balancing requests across all nodes that host replicas for the collection you are querying, as Solr's `CloudSolrClient` will do. If not load-balancing, this feature can introduce a hotspot in the cluster since queries won't be evenly distributed across the cluster.
+`replica.base`::
+Applied after sorting by inherent replica attributes, this property defines a fallback ordering among sets of preference-equivalent replicas; if specified, only one value may be specified for this property, and it must be specified last.
++
+`random`, the default, randomly shuffles replicas for each request. This distributes requests evenly, but can result in sub-optimal cache usage for shards with replication factor > 1.
++
+`stable:dividend:_paramName_` parses an integer from the value associated with the given param name; this integer is used as the dividend (mod equivalent replica count) to determine (via list rotation) order of preference among equivalent replicas.
++
+`stable[:hash[:_paramName_]]` the string value associated with the given param name is hashed to a dividend that is used to determine replica preference order (analogous to the explicit `dividend` property above); `_paramName_` defaults to `q` if not specified, providing stable routing keyed to the string value of the "main query". Note that this may be inappropriate for some use cases (e.g., static main queries that leverage parameter substitution)
+
Examples:
+* Prefer stable routing (keyed to client "sessionId" param) among otherwise equivalent replicas:
+ `shards.preference=replica.base:stable:hash:sessionId&sessionId=abc123`
+
* Prefer PULL replicas:
`shards.preference=replica.type:PULL`
diff --git a/solr/solr-ref-guide/src/format-of-solr-xml.adoc b/solr/solr-ref-guide/src/format-of-solr-xml.adoc
index 199ae07..5d95efd 100644
--- a/solr/solr-ref-guide/src/format-of-solr-xml.adoc
+++ b/solr/solr-ref-guide/src/format-of-solr-xml.adoc
@@ -203,6 +203,20 @@ If the threadpool uses a backing queue, what is its maximum size to use direct h
`fairnessPolicy`::
A boolean to configure if the threadpool favors fairness over throughput. Default is false to favor throughput.
+`replicaRouting`::
+A NamedList specifying replica routing preference configuration. This may be used to select and configure replica routing preferences. `default=true` may be used to set the default base replica routing preference. Only positive default status assertions are respected; i.e., `default=false` has no effect. If no explicit default base replica routing preference is configured, the implicit default will be `random`.
+----
+<shardHandlerFactory class="HttpShardHandlerFactory">
+ <lst name="replicaRouting">
+ <lst name="stable">
+ <bool name="default">true</bool>
+ <str name="dividend">routingDividend</str>
+ <str name="hash">q</str>
+ </lst>
+ </lst>
+</shardHandlerFactory>
+----
+Replica routing may also be specified (overriding defaults) per-request, via the `shards.preference` request parameter. If a request contains both dividend param and hash param, dividend param takes priority for routing. For configuring `stable` routing, the `hash` param implicitly defaults to a hash of the String value of the main query param (i.e., `q`). `dividend` param must be configured explicitly; there is no implicit default. If only `dividend` routing is desired, `hash` may be ex [...]
=== The <metrics> Element
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
index 7844e03..a2f1563 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
@@ -66,9 +66,24 @@ public interface ShardParams {
/** Node with same system property sort rule */
String SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP = "node.sysprop";
+ /** Replica base/fallback sort rule */
+ String SHARDS_PREFERENCE_REPLICA_BASE = "replica.base";
+
/** Value denoting local replicas */
String REPLICA_LOCAL = "local";
+ /** Value denoting randomized replica sort */
+ String REPLICA_RANDOM = "random";
+
+ /** Value denoting stable replica sort */
+ String REPLICA_STABLE = "stable";
+
+ /** configure dividend param for stable replica sort */
+ String ROUTING_DIVIDEND = "dividend";
+
+ /** configure hash param for stable replica sort */
+ String ROUTING_HASH = "hash";
+
String _ROUTE_ = "_route_";
/** Force a single-pass distributed query? (true/false) */