You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/03/25 06:57:00 UTC
[lucene-solr] branch branch_6_6 updated: SOLR-12770: Make it
possible to configure a host whitelist for distributed search
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch branch_6_6
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_6_6 by this push:
new 5ef1e56 SOLR-12770: Make it possible to configure a host whitelist for distributed search
5ef1e56 is described below
commit 5ef1e568f1fdfbb8a46470fa3178f0ce6978ea1b
Author: Tomas Fernandez Lobbe <tf...@apache.org>
AuthorDate: Tue Jan 15 11:44:57 2019 -0800
SOLR-12770: Make it possible to configure a host whitelist for distributed search
---
solr/CHANGES.txt | 9 +
solr/bin/solr.in.cmd | 6 +
solr/bin/solr.in.sh | 5 +
.../solr/handler/component/HttpShardHandler.java | 38 ++-
.../handler/component/HttpShardHandlerFactory.java | 173 +++++++++++++-
.../solr/handler/component/TermsComponent.java | 70 +++++-
solr/core/src/test-files/solr/solr.xml | 1 +
.../test/org/apache/solr/TestTolerantSearch.java | 2 +
.../apache/solr/cloud/MultiSolrCloudTestCase.java | 110 +++++++++
.../component/DistributedDebugComponentTest.java | 2 +
.../handler/component/ShardsWhitelistTest.java | 263 +++++++++++++++++++++
.../component/TestHttpShardHandlerFactory.java | 247 +++++++++++++++++++
.../org/apache/solr/search/TestSmileRequest.java | 2 +
.../solr/search/facet/TestJsonFacetRefinement.java | 2 +
.../apache/solr/search/facet/TestJsonFacets.java | 29 ++-
.../apache/solr/search/json/TestJsonRequest.java | 2 +
solr/server/solr/solr.xml | 1 +
solr/solr-ref-guide/src/distributed-requests.adoc | 6 +-
.../distributed-search-with-index-sharding.adoc | 3 +
solr/solr-ref-guide/src/the-terms-component.adoc | 7 +
solr/solrj/src/test-files/solrj/solr/solr.xml | 1 +
.../apache/solr/BaseDistributedSearchTestCase.java | 12 +
.../src/java/org/apache/solr/SolrTestCaseHS.java | 22 ++
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 15 +-
.../apache/solr/cloud/MiniSolrCloudCluster.java | 3 +
.../org/apache/solr/cloud/SolrCloudTestCase.java | 4 +-
26 files changed, 1014 insertions(+), 21 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5ab688c..354c7cd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -29,6 +29,12 @@ Apache UIMA 2.3.1
Apache ZooKeeper 3.4.10
Jetty 9.3.14.v20161028
+Upgrade Notes
+----------------------
+
+* SOLR-12770: The 'shards' parameter handling logic changes to use a new config element to determine what hosts can be
+ requested. Please see Apache Solr Reference Guide chapter "Distributed Requests" for details, as well as SOLR-12770.
+
Bug Fixes
----------------------
@@ -38,6 +44,9 @@ Bug Fixes
* SOLR-12514: Rule-base Authorization plugin skips authorization if querying node does not have collection
replica (noble)
+* SOLR-12770: Make it possible to configure a host whitelist for distributed search
+ (Christine Poerschke, janhoy, Erick Erickson, Tomás Fernández Löbbe)
+
================== 6.6.5 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index 4e22742..d25b238 100644
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -121,3 +121,9 @@ REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCrede
REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
REM set SOLR_OPTS=%SOLR_OPTS% %SOLR_ZK_CREDS_AND_ACLS%
+
+REM When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the
+REM list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml,
+REM or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively
+REM host checking can be disabled by using the system property "solr.disable.shardsWhitelist"
+REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984"
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index 1d090f5..b439b06 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -139,3 +139,8 @@
# -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD"
#SOLR_OPTS="$SOLR_OPTS $SOLR_ZK_CREDS_AND_ACLS"
+# When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the
+# list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml,
+# or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively
+# host checking can be disabled by using the system property "solr.disable.shardsWhitelist"
+#SOLR_OPTS="$SOLR_OPTS -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984"
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
index 8c0a9cb..6dbed72 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
@@ -19,6 +19,7 @@ package org.apache.solr.handler.component;
import java.lang.invoke.MethodHandles;
import java.net.ConnectException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -42,6 +43,7 @@ import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@@ -79,7 +81,6 @@ public class HttpShardHandler extends ShardHandler {
// This is primarily to keep track of what order we should use to query the replicas of a shard
// so that we use the same replica for all phases of a distributed request.
shardToURLs = new HashMap<>();
-
}
@@ -340,6 +341,13 @@ public class HttpShardHandler extends ShardHandler {
rb.shards = new String[rb.slices.length];
}
+ HttpShardHandlerFactory.WhitelistHostChecker hostChecker = httpShardHandlerFactory.getWhitelistHostChecker();
+ if (shards != null && zkController == null && hostChecker.isWhitelistHostCheckingEnabled() && !hostChecker.hasExplicitWhitelist()) {
+ throw new SolrException(ErrorCode.FORBIDDEN, "HttpShardHandlerFactory "+HttpShardHandlerFactory.INIT_SHARDS_WHITELIST
+ +" not configured but required (in lieu of ZkController and ClusterState) when using the '"+ShardParams.SHARDS+"' parameter."
+ +HttpShardHandlerFactory.SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE);
+ }
+
//
// Map slices to shards
//
@@ -360,10 +368,21 @@ public class HttpShardHandler extends ShardHandler {
if (shortCircuit) {
rb.isDistrib = false;
rb.shortCircuitedURL = ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), coreDescriptor.getName());
+ if (hostChecker.isWhitelistHostCheckingEnabled() && hostChecker.hasExplicitWhitelist()) {
+ /*
+ * We only need to check the host whitelist if there is an explicit whitelist (other than all the live nodes)
+ * when the "shards" indicate cluster state elements only
+ */
+ hostChecker.checkWhitelist(clusterState, shards, Arrays.asList(rb.shortCircuitedURL));
+ }
return;
}
// We shouldn't need to do anything to handle "shard.rows" since it was previously meant to be an optimization?
}
+
+ if (clusterState == null && zkController != null) {
+ clusterState = zkController.getClusterState();
+ }
for (int i=0; i<rb.shards.length; i++) {
@@ -371,9 +390,9 @@ public class HttpShardHandler extends ShardHandler {
if (rb.shards[i] != null) {
shardUrls = StrUtils.splitSmart(rb.shards[i], "|", true);
replicaListTransformer.transform(shardUrls);
+ hostChecker.checkWhitelist(clusterState, shards, shardUrls);
} else {
- if (clusterState == null) {
- clusterState = zkController.getClusterState();
+ if (slices == null) {
slices = clusterState.getSlicesMap(cloudDescriptor.getCollectionName());
}
String sliceName = rb.slices[i];
@@ -406,6 +425,14 @@ public class HttpShardHandler extends ShardHandler {
shardUrls.add(url);
}
+ if (hostChecker.isWhitelistHostCheckingEnabled() && hostChecker.hasExplicitWhitelist()) {
+ /*
+ * We only need to check the host whitelist if there is an explicit whitelist (other than all the live nodes)
+ * when the "shards" indicate cluster state elements only
+ */
+ hostChecker.checkWhitelist(clusterState, shards, shardUrls);
+ }
+
if (shardUrls.isEmpty()) {
boolean tolerant = rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false);
if (!tolerant) {
@@ -428,6 +455,11 @@ public class HttpShardHandler extends ShardHandler {
}
rb.shards[i] = sliceShardsStr.toString();
}
+ } else {
+ if (shards != null) {
+ // No cloud, verbatim check of shards
+ hostChecker.checkWhitelist(shards, new ArrayList<>(Arrays.asList(shards.split("[,|]"))));
+ }
}
String shards_rows = params.get(ShardParams.SHARDS_ROWS);
if(shards_rows != null) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
index 36a1144..a17cc0a 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
@@ -22,6 +22,12 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Random;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletionService;
@@ -29,7 +35,7 @@ import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
-
+import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -44,9 +50,12 @@ import org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
@@ -107,6 +116,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
int connectionsEvictorSleepDelay = UpdateShardHandlerConfig.DEFAULT_UPDATECONNECTIONSEVICTORSLEEPDELAY;
protected UpdateShardHandler.IdleConnectionsEvictor idleConnectionsEvictor;
+ private WhitelistHostChecker whitelistHostChecker = null;
private String scheme = null;
@@ -142,6 +152,12 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
static final String MAX_CONNECTION_IDLE_TIME = "maxConnectionIdleTime";
+ public static final String INIT_SHARDS_WHITELIST = "shardsWhitelist";
+
+ static final String INIT_SOLR_DISABLE_SHARDS_WHITELIST = "solr.disable." + INIT_SHARDS_WHITELIST;
+
+ static final String SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE = " set -D"+INIT_SOLR_DISABLE_SHARDS_WHITELIST+"=true to disable shards whitelist checks";
+
/**
* Get {@link ShardHandler} that uses the default http client.
*/
@@ -157,6 +173,24 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
return new HttpShardHandler(this, httpClient);
}
+ /**
+ * Returns this Factory's {@link WhitelistHostChecker}.
+ * This method can be overridden to change the checker implementation.
+ */
+ public WhitelistHostChecker getWhitelistHostChecker() {
+ return this.whitelistHostChecker;
+ }
+
+ @Deprecated // For temporary use by the TermsComponent only.
+ static boolean doGetDisableShardsWhitelist() {
+ return getDisableShardsWhitelist();
+ }
+
+
+ private static boolean getDisableShardsWhitelist() {
+ return Boolean.getBoolean(INIT_SOLR_DISABLE_SHARDS_WHITELIST);
+ }
+
@Override
public void init(PluginInfo info) {
StringBuilder sb = new StringBuilder();
@@ -186,6 +220,9 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
this.connectionsEvictorSleepDelay = getParameter(args, CONNECTIONS_EVICTOR_SLEEP_DELAY, connectionsEvictorSleepDelay, sb);
this.maxConnectionIdleTime = getParameter(args, MAX_CONNECTION_IDLE_TIME, maxConnectionIdleTime, sb);
+ this.whitelistHostChecker = new WhitelistHostChecker(args == null? null: (String) args.get(INIT_SHARDS_WHITELIST), !getDisableShardsWhitelist());
+ log.info("Host whitelist initialized: {}", this.whitelistHostChecker);
+
log.debug("created with {}",sb);
// magic sysprop to make tests reproducible: set by SolrTestCaseJ4.
@@ -430,4 +467,138 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
manager.registry(registry),
SolrMetricManager.mkName("httpShardExecutor", expandedScope, "threadPool"));
}
+
+ /**
+ * Class used to validate the hosts in the "shards" parameter when doing a distributed
+ * request
+ */
+ public static class WhitelistHostChecker {
+
+ /**
+ * List of the whitelisted hosts. Elements in the list will be host:port (no protocol or context)
+ */
+ private final Set<String> whitelistHosts;
+
+ /**
+ * Indicates whether host checking is enabled
+ */
+ private final boolean whitelistHostCheckingEnabled;
+
+ public WhitelistHostChecker(String whitelistStr, boolean enabled) {
+ this.whitelistHosts = implGetShardsWhitelist(whitelistStr);
+ this.whitelistHostCheckingEnabled = enabled;
+ }
+
+ final static Set<String> implGetShardsWhitelist(final String shardsWhitelist) {
+ if (shardsWhitelist != null && !shardsWhitelist.isEmpty()) {
+ return StrUtils.splitSmart(shardsWhitelist, ',')
+ .stream()
+ .map(String::trim)
+ .map((hostUrl) -> {
+ URL url;
+ try {
+ if (!hostUrl.startsWith("http://") && !hostUrl.startsWith("https://")) {
+ // It doesn't really matter which protocol we set here because we are not going to use it. We just need a full URL.
+ url = new URL("http://" + hostUrl);
+ } else {
+ url = new URL(hostUrl);
+ }
+ } catch (MalformedURLException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist, e);
+ }
+ if (url.getHost() == null || url.getPort() < 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid URL syntax in \"" + INIT_SHARDS_WHITELIST + "\": " + shardsWhitelist);
+ }
+ return url.getHost() + ":" + url.getPort();
+ }).collect(Collectors.toSet());
+ }
+ return null;
+ }
+
+
+ /**
+ * @see #checkWhitelist(ClusterState, String, List)
+ */
+ protected void checkWhitelist(String shardsParamValue, List<String> shardUrls) {
+ checkWhitelist(null, shardsParamValue, shardUrls);
+ }
+
+ /**
+ * Checks that all the hosts for all the shards requested in shards parameter exist in the configured whitelist
+ * or in the ClusterState (in case of cloud mode)
+ *
+ * @param clusterState The up to date ClusterState, can be null in case of non-cloud mode
+ * @param shardsParamValue The original shards parameter
+ * @param shardUrls The list of cores generated from the shards parameter.
+ */
+ protected void checkWhitelist(ClusterState clusterState, String shardsParamValue, List<String> shardUrls) {
+ if (!whitelistHostCheckingEnabled) {
+ return;
+ }
+ Set<String> localWhitelistHosts;
+ if (whitelistHosts == null && clusterState != null) {
+ // TODO: We could implement caching, based on the version of the live_nodes znode
+ localWhitelistHosts = generateWhitelistFromLiveNodes(clusterState);
+ } else if (whitelistHosts != null) {
+ localWhitelistHosts = whitelistHosts;
+ } else {
+ localWhitelistHosts = Collections.emptySet();
+ }
+
+ shardUrls.stream().map(String::trim).forEach((shardUrl) -> {
+ URL url;
+ try {
+ if (!shardUrl.startsWith("http://") && !shardUrl.startsWith("https://")) {
+ // It doesn't really matter which protocol we set here because we are not going to use it. We just need a full URL.
+ url = new URL("http://" + shardUrl);
+ } else {
+ url = new URL(shardUrl);
+ }
+ } catch (MalformedURLException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue, e);
+ }
+ if (url.getHost() == null || url.getPort() < 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid URL syntax in \"shards\" parameter: " + shardsParamValue);
+ }
+ if (!localWhitelistHosts.contains(url.getHost() + ":" + url.getPort())) {
+ log.warn("The '"+ShardParams.SHARDS+"' parameter value '"+shardsParamValue+"' contained value(s) not on the shards whitelist ("+localWhitelistHosts+"), shardUrl:" + shardUrl);
+ throw new SolrException(ErrorCode.FORBIDDEN,
+ "The '"+ShardParams.SHARDS+"' parameter value '"+shardsParamValue+"' contained value(s) not on the shards whitelist. shardUrl:" + shardUrl + "." +
+ HttpShardHandlerFactory.SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE);
+ }
+ });
+ }
+
+ Set<String> generateWhitelistFromLiveNodes(ClusterState clusterState) {
+ return clusterState
+ .getLiveNodes()
+ .stream()
+ .map((liveNode) -> liveNode.substring(0, liveNode.indexOf('_')))
+ .collect(Collectors.toSet());
+ }
+
+ public boolean hasExplicitWhitelist() {
+ return this.whitelistHosts != null;
+ }
+
+ public boolean isWhitelistHostCheckingEnabled() {
+ return whitelistHostCheckingEnabled;
+ }
+
+ /**
+ * Only to be used by tests
+ */
+ @VisibleForTesting
+ Set<String> getWhitelistHosts() {
+ return this.whitelistHosts;
+ }
+
+ @Override
+ public String toString() {
+ return "WhitelistHostChecker [whitelistHosts=" + whitelistHosts + ", whitelistHostCheckingEnabled="
+ + whitelistHostCheckingEnabled + "]";
+ }
+
+ }
+
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
index 8a735d1..ebe7125 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
@@ -15,30 +15,45 @@
* limitations under the License.
*/
package org.apache.solr.handler.component;
-import org.apache.lucene.index.*;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.StringHelper;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.*;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.TermsParams;
+import org.apache.solr.handler.component.HttpShardHandlerFactory.WhitelistHostChecker;
+import org.apache.solr.request.SimpleFacets.CountPair;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.StrField;
-import org.apache.solr.request.SimpleFacets.CountPair;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.BoundedTreeSet;
import org.apache.solr.client.solrj.response.TermsResponse;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.regex.Pattern;
-
/**
* Return TermEnum information, useful for things like auto suggest.
*
@@ -62,6 +77,20 @@ public class TermsComponent extends SearchComponent {
public static final int UNLIMITED_MAX_COUNT = -1;
public static final String COMPONENT_NAME = "terms";
+ // This needs to be created here too, because Solr doesn't call init(...) on default components. Bug?
+ private WhitelistHostChecker whitelistHostChecker = new WhitelistHostChecker(
+ null,
+ !HttpShardHandlerFactory.doGetDisableShardsWhitelist());
+
+ @Override
+ public void init( NamedList args )
+ {
+ super.init(args);
+ whitelistHostChecker = new WhitelistHostChecker(
+ (String) args.get(HttpShardHandlerFactory.INIT_SHARDS_WHITELIST),
+ !HttpShardHandlerFactory.doGetDisableShardsWhitelist());
+ }
+
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
@@ -81,10 +110,30 @@ public class TermsComponent extends SearchComponent {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shards.qt parameter specified");
}
List<String> lst = StrUtils.splitSmart(shards, ",", true);
+ checkShardsWhitelist(rb, lst);
rb.shards = lst.toArray(new String[lst.size()]);
}
}
+ protected void checkShardsWhitelist(final ResponseBuilder rb, final List<String> lst) {
+ final List<String> urls = new LinkedList<String>();
+ for (final String ele : lst) {
+ urls.addAll(StrUtils.splitSmart(ele, '|'));
+ }
+
+ if (whitelistHostChecker.isWhitelistHostCheckingEnabled() && rb.req.getCore().getCoreContainer().getZkController() == null && !whitelistHostChecker.hasExplicitWhitelist()) {
+ throw new SolrException(ErrorCode.FORBIDDEN, "TermsComponent "+HttpShardHandlerFactory.INIT_SHARDS_WHITELIST
+ +" not configured but required when using the '"+ShardParams.SHARDS+"' parameter with the TermsComponent."
+ +HttpShardHandlerFactory.SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE);
+ } else {
+ ClusterState cs = null;
+ if (rb.req.getCore().getCoreContainer().getZkController() != null) {
+ cs = rb.req.getCore().getCoreContainer().getZkController().getClusterState();
+ }
+ whitelistHostChecker.checkWhitelist(cs, urls.toString(), urls);
+ }
+ }
+
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
@@ -601,4 +650,5 @@ public class TermsComponent extends SearchComponent {
public Category getCategory() {
return Category.QUERY;
}
+
}
diff --git a/solr/core/src/test-files/solr/solr.xml b/solr/core/src/test-files/solr/solr.xml
index 526dffa..51083a9 100644
--- a/solr/core/src/test-files/solr/solr.xml
+++ b/solr/core/src/test-files/solr/solr.xml
@@ -29,6 +29,7 @@
<str name="urlScheme">${urlScheme:}</str>
<int name="socketTimeout">${socketTimeout:90000}</int>
<int name="connTimeout">${connTimeout:15000}</int>
+ <str name="shardsWhitelist">${solr.tests.shardsWhitelist:}</str>
</shardHandlerFactory>
<transientCoreCacheFactory name="transientCoreCacheFactory" class="TransientSolrCoreCacheFactoryDefault">
diff --git a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java
index cb485d0..d869cd3 100644
--- a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java
+++ b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java
@@ -56,6 +56,7 @@ public class TestTolerantSearch extends SolrJettyTestBase {
@BeforeClass
public static void createThings() throws Exception {
+ systemSetPropertySolrDisableShardsWhitelist("true");
solrHome = createSolrHome();
createJetty(solrHome.getAbsolutePath());
String url = jetty.getBaseUrl().toString();
@@ -105,6 +106,7 @@ public class TestTolerantSearch extends SolrJettyTestBase {
jetty.stop();
jetty=null;
resetExceptionIgnores();
+ systemClearPropertySolrDisableShardsWhitelist();
}
@SuppressWarnings("unchecked")
diff --git a/solr/core/src/test/org/apache/solr/cloud/MultiSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/MultiSolrCloudTestCase.java
new file mode 100644
index 0000000..8a4b211
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/MultiSolrCloudTestCase.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.junit.AfterClass;
+
+/**
+ * Base class for tests that require more than one SolrCloud
+ *
+ * Derived tests should call {@link #doSetupClusters(String[], Function, BiConsumer)} in a {@code BeforeClass}
+ * static method. This configures and starts the {@link MiniSolrCloudCluster} instances, available
+ * via the {@code clusterId2cluster} variable. The clusters' shutdown is handled automatically.
+ */
+public abstract class MultiSolrCloudTestCase extends SolrTestCaseJ4 {
+
+ protected static Map<String,MiniSolrCloudCluster> clusterId2cluster = new HashMap<String,MiniSolrCloudCluster>();
+
+ protected static abstract class DefaultClusterCreateFunction implements Function<String,MiniSolrCloudCluster> {
+
+ public DefaultClusterCreateFunction() {
+ }
+
+ protected abstract int nodesPerCluster(String clusterId);
+
+ @Override
+ public MiniSolrCloudCluster apply(String clusterId) {
+ try {
+ new SolrCloudTestCase
+ .Builder(nodesPerCluster(clusterId), createTempDir())
+ .addConfig("conf", configset("cloud-dynamic"))
+ .configure();
+
+ final MiniSolrCloudCluster cluster = SolrCloudTestCase.cluster;
+ return cluster;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ protected static abstract class DefaultClusterInitFunction implements BiConsumer<String,MiniSolrCloudCluster> {
+
+ final private int numShards;
+ final private int numReplicas;
+ final private int maxShardsPerNode;
+
+ public DefaultClusterInitFunction(int numShards, int numReplicas, int maxShardsPerNode) {
+ this.numShards = numShards;
+ this.numReplicas = numReplicas;
+ this.maxShardsPerNode = maxShardsPerNode;
+ }
+
+ protected void doAccept(String collection, MiniSolrCloudCluster cluster) {
+ try {
+ CollectionAdminRequest
+ .createCollection(collection, "conf", numShards, numReplicas)
+ .setMaxShardsPerNode(maxShardsPerNode)
+ .processAndWait(cluster.getSolrClient(), SolrCloudTestCase.DEFAULT_TIMEOUT);
+
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collection, cluster.getSolrClient().getZkStateReader(), false, true, SolrCloudTestCase.DEFAULT_TIMEOUT);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ protected static void doSetupClusters(final String[] clusterIds,
+ final Function<String,MiniSolrCloudCluster> createFunc,
+ final BiConsumer<String,MiniSolrCloudCluster> initFunc) throws Exception {
+
+ for (final String clusterId : clusterIds) {
+ assertFalse("duplicate clusterId "+clusterId, clusterId2cluster.containsKey(clusterId));
+ MiniSolrCloudCluster cluster = createFunc.apply(clusterId);
+ initFunc.accept(clusterId, cluster);
+ clusterId2cluster.put(clusterId, cluster);
+ }
+ }
+
+ @AfterClass
+ public static void shutdownCluster() throws Exception {
+ for (MiniSolrCloudCluster cluster : clusterId2cluster.values()) {
+ cluster.shutdown();
+ }
+ clusterId2cluster.clear();
+ }
+
+}
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java
index b447668..93b86d9 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedDebugComponentTest.java
@@ -63,6 +63,7 @@ public class DistributedDebugComponentTest extends SolrJettyTestBase {
@BeforeClass
public static void createThings() throws Exception {
+ systemSetPropertySolrDisableShardsWhitelist("true");
solrHome = createSolrHome();
createJetty(solrHome.getAbsolutePath());
String url = jetty.getBaseUrl().toString();
@@ -105,6 +106,7 @@ public class DistributedDebugComponentTest extends SolrJettyTestBase {
jetty.stop();
jetty=null;
resetExceptionIgnores();
+ systemClearPropertySolrDisableShardsWhitelist();
}
@Test
diff --git a/solr/core/src/test/org/apache/solr/handler/component/ShardsWhitelistTest.java b/solr/core/src/test/org/apache/solr/handler/component/ShardsWhitelistTest.java
new file mode 100644
index 0000000..bfca21b
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/ShardsWhitelistTest.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.nullValue;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.MultiSolrCloudTestCase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ShardsWhitelistTest extends MultiSolrCloudTestCase {
+
+ /**
+ * The cluster with this key will include an explicit list of host whitelisted (all hosts in both the clusters)
+ */
+ private static final String EXPLICIT_CLUSTER_KEY = "explicitCluster";
+ /**
+ * The cluster with this key will not include an explicit list of host whitelisted, will rely on live_nodes
+ */
+ private static final String IMPLICIT_CLUSTER_KEY = "implicitCluster";
+ private static final String EXPLICIT_WHITELIST_PROPERTY = "solr.tests.ShardsWhitelistTest.explicitWhitelist.";
+ protected static final String COLLECTION_NAME = "ShardsWhitelistTestCollection";
+
+ private static int numShards;
+ private static int numReplicas;
+ private static int maxShardsPerNode;
+ private static int nodesPerCluster;
+
+ private static void appendClusterNodes(final StringBuilder sb, final String delimiter,
+ final MiniSolrCloudCluster cluster) {
+ cluster.getJettySolrRunners().forEach((jetty) -> sb.append(jetty.getBaseUrl().toString() + delimiter));
+ }
+
+ @BeforeClass
+ public static void setupClusters() throws Exception {
+
+ final String[] clusterIds = new String[] {IMPLICIT_CLUSTER_KEY, EXPLICIT_CLUSTER_KEY};
+
+ numShards = 2; // +random().nextInt(2);
+ numReplicas = 1; // +random().nextInt(2);
+ maxShardsPerNode = 1; // +random().nextInt(2);
+ nodesPerCluster = (numShards * numReplicas + (maxShardsPerNode - 1)) / maxShardsPerNode;
+
+ final StringBuilder sb = new StringBuilder();
+
+ doSetupClusters(clusterIds,
+ new DefaultClusterCreateFunction() {
+
+ @Override
+ public MiniSolrCloudCluster apply(String clusterId) {
+ try {
+ new SolrCloudTestCase.Builder(nodesPerCluster(clusterId),
+ createTempDir())
+ .addConfig("conf", configset("cloud-dynamic"))
+ .withSolrXml(MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML.replace(
+ MiniSolrCloudCluster.SOLR_TESTS_SHARDS_WHITELIST, EXPLICIT_WHITELIST_PROPERTY + clusterId))
+ .configure();
+ final MiniSolrCloudCluster cluster = SolrCloudTestCase.cluster;
+ return cluster;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ protected int nodesPerCluster(String clusterId) {
+ return nodesPerCluster;
+ }
+ },
+ new DefaultClusterInitFunction(numShards, numReplicas, maxShardsPerNode) {
+ @Override
+ public void accept(String clusterId, MiniSolrCloudCluster cluster) {
+ appendClusterNodes(sb, ",", cluster);
+ if (clusterId.equals(EXPLICIT_CLUSTER_KEY)) {
+ System.setProperty(EXPLICIT_WHITELIST_PROPERTY + clusterId, sb.toString());
+ for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
+ try {
+ runner.stop();
+ runner.start(true);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to restart runner", e);
+ }
+ }
+ }
+ doAccept(COLLECTION_NAME, cluster);
+ }
+ });
+ }
+
+ @AfterClass
+ public static void afterTests() {
+ System.clearProperty(EXPLICIT_WHITELIST_PROPERTY + EXPLICIT_CLUSTER_KEY);
+ }
+
+ private HttpShardHandlerFactory getShardHandlerFactory(String clusterId) {
+ return (HttpShardHandlerFactory) clusterId2cluster.get(clusterId).getJettySolrRunner(0).getCoreContainer()
+ .getShardHandlerFactory();
+ }
+
+ @Test
+ public void test() throws Exception {
+ assertThat(getShardHandlerFactory(EXPLICIT_CLUSTER_KEY).getWhitelistHostChecker().getWhitelistHosts(), notNullValue());
+ assertThat(getShardHandlerFactory(IMPLICIT_CLUSTER_KEY).getWhitelistHostChecker().getWhitelistHosts(), nullValue());
+
+ assertThat(getShardHandlerFactory(EXPLICIT_CLUSTER_KEY).getWhitelistHostChecker().hasExplicitWhitelist(), is(true));
+ assertThat(getShardHandlerFactory(IMPLICIT_CLUSTER_KEY).getWhitelistHostChecker().hasExplicitWhitelist(), is(false));
+ for (MiniSolrCloudCluster cluster : clusterId2cluster.values()) {
+ for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
+ URI uri = runner.getBaseUrl().toURI();
+ assertTrue(getShardHandlerFactory(EXPLICIT_CLUSTER_KEY).getWhitelistHostChecker().getWhitelistHosts().
+ contains(uri.getHost() + ":" + uri.getPort()));
+ }
+ }
+
+ MiniSolrCloudCluster implicitCluster = clusterId2cluster.get(IMPLICIT_CLUSTER_KEY);
+ MiniSolrCloudCluster explicitCluster = clusterId2cluster.get(EXPLICIT_CLUSTER_KEY);
+
+ for (Map.Entry<String,MiniSolrCloudCluster> entry : clusterId2cluster.entrySet()) {
+ List<SolrInputDocument> docs = new ArrayList<>(10);
+ for (int i = 0; i < 10; i++) {
+ docs.add(new SolrInputDocument("id", entry.getKey() + i));
+ }
+ MiniSolrCloudCluster cluster = entry.getValue();
+ cluster.getSolrClient().add(COLLECTION_NAME, docs);
+ cluster.getSolrClient().commit(COLLECTION_NAME, true, true);
+
+ // test using ClusterState elements
+ assertThat("No shards specified, should work in both clusters",
+ numDocs("*:*", null, cluster), is(10));
+ assertThat("Both shards specified, should work in both clusters",
+ numDocs("*:*", "shard1,shard2", cluster), is(10));
+ assertThat("Both shards specified with collection name, should work in both clusters",
+ numDocs("*:*", COLLECTION_NAME + "_shard1", cluster), is(numDocs("*:*", "shard1", cluster)));
+
+ // test using explicit urls from within the cluster
+ assertThat("Shards has the full URLs, should be allowed since they are internal. Cluster=" + entry.getKey(),
+ numDocs("*:*", getShardUrl("shard1", cluster) + "," + getShardUrl("shard2", cluster), cluster), is(10));
+ assertThat("Full URL without scheme",
+ numDocs("*:*", getShardUrl("shard1", cluster).replaceAll("http://", "") + ","
+ + getShardUrl("shard2", cluster).replaceAll("http://", ""), cluster),
+ is(10));
+
+ // Mix shards with URLs
+ assertThat("Mix URL and cluster state object",
+ numDocs("*:*", "shard1," + getShardUrl("shard2", cluster), cluster), is(10));
+ assertThat("Mix URL and cluster state object",
+ numDocs("*:*", getShardUrl("shard1", cluster) + ",shard2", cluster), is(10));
+ }
+
+ // explicit whitelist includes all the nodes in both clusters. Requests should be allowed to go through
+ assertThat("A request to the explicit cluster with shards that point to the implicit one",
+ numDocs(
+ "id:implicitCluster*",
+ getShardUrl("shard1", implicitCluster) + "," + getShardUrl("shard2", implicitCluster),
+ explicitCluster),
+ is(10));
+
+ assertThat("A request to the explicit cluster with shards that point to the both clusters",
+ numDocs(
+ "*:*",
+ getShardUrl("shard1", implicitCluster)
+ + "," + getShardUrl("shard2", implicitCluster)
+ + "," + getShardUrl("shard1", explicitCluster)
+ + "," + getShardUrl("shard2", explicitCluster),
+ explicitCluster),
+ is(20));
+
+ // Implicit shouldn't allow requests to the other cluster
+ assertForbidden("id:explicitCluster*",
+ getShardUrl("shard1", explicitCluster) + "," + getShardUrl("shard2", explicitCluster),
+ implicitCluster);
+
+ assertForbidden("id:explicitCluster*",
+ "shard1," + getShardUrl("shard2", explicitCluster),
+ implicitCluster);
+
+ assertForbidden("id:explicitCluster*",
+ getShardUrl("shard1", explicitCluster) + ",shard2",
+ implicitCluster);
+
+ assertForbidden("id:explicitCluster*",
+ getShardUrl("shard1", explicitCluster),
+ implicitCluster);
+
+ assertThat("A typical internal request, should be handled locally",
+ numDocs(
+ "id:explicitCluster*",
+ null,
+ implicitCluster,
+ "distrib", "false",
+ "shard.url", getShardUrl("shard2", explicitCluster),
+ "shards.purpose", "64",
+ "isShard", "true"),
+ is(0));
+ }
+
+ private void assertForbidden(String query, String shards, MiniSolrCloudCluster cluster) throws IOException {
+ ignoreException("not on the shards whitelist");
+ try {
+ numDocs(
+ query,
+ shards,
+ cluster);
+ fail("Expecting failure for shards parameter: '" + shards + "'");
+ } catch (SolrServerException e) {
+ assertThat(e.getCause(), instanceOf(SolrException.class));
+ assertThat(((SolrException) e.getCause()).code(), is(SolrException.ErrorCode.FORBIDDEN.code));
+ assertTrue(((SolrException) e.getCause()).getMessage().contains("not on the shards whitelist"));
+ }
+ unIgnoreException("not on the shards whitelist");
+ }
+
+ private String getShardUrl(String shardName, MiniSolrCloudCluster cluster) {
+ return cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION_NAME)
+ .getSlice(shardName).getReplicas().iterator().next().getCoreUrl();
+ }
+
+ private int numDocs(String queryString, String shardsParamValue, MiniSolrCloudCluster cluster, String... otherParams)
+ throws SolrServerException, IOException {
+ SolrQuery q = new SolrQuery(queryString);
+ if (shardsParamValue != null) {
+ q.set("shards", shardsParamValue);
+ }
+ if (otherParams != null) {
+ assert otherParams.length % 2 == 0;
+ for (int i = 0; i < otherParams.length; i += 2) {
+ q.set(otherParams[i], otherParams[i + 1]);
+ }
+ }
+ return (int) cluster.getSolrClient().query(COLLECTION_NAME, q).getResults().getNumFound();
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
new file mode 100644
index 0000000..70cf996
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.component.HttpShardHandlerFactory.WhitelistHostChecker;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests specifying a custom ShardHandlerFactory
+ */
+public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 {
+
+ private static final String LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE = "solr.tests.loadBalancerRequestsMinimumAbsolute";
+ private static final String LOAD_BALANCER_REQUESTS_MAX_FRACTION = "solr.tests.loadBalancerRequestsMaximumFraction";
+ private static final String SHARDS_WHITELIST = "solr.tests.shardsWhitelist";
+
+ private static int expectedLoadBalancerRequestsMinimumAbsolute = 0;
+ private static float expectedLoadBalancerRequestsMaximumFraction = 1.0f;
+
+ @BeforeClass
+ public static void beforeTests() throws Exception {
+ expectedLoadBalancerRequestsMinimumAbsolute = random().nextInt(3); // 0 .. 2
+ expectedLoadBalancerRequestsMaximumFraction = (1+random().nextInt(10))/10f; // 0.1 .. 1.0
+ System.setProperty(LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE, Integer.toString(expectedLoadBalancerRequestsMinimumAbsolute));
+ System.setProperty(LOAD_BALANCER_REQUESTS_MAX_FRACTION, Float.toString(expectedLoadBalancerRequestsMaximumFraction));
+
+ }
+
+ @AfterClass
+ public static void afterTests() {
+ System.clearProperty(LOAD_BALANCER_REQUESTS_MIN_ABSOLUTE);
+ System.clearProperty(LOAD_BALANCER_REQUESTS_MAX_FRACTION);
+ }
+
+ @Test
+ public void getShardsWhitelist() throws Exception {
+ System.setProperty(SHARDS_WHITELIST, "http://abc:8983/,http://def:8984/,");
+ final Path home = Paths.get(TEST_HOME());
+ CoreContainer cc = null;
+ ShardHandlerFactory factory = null;
+ try {
+ cc = CoreContainer.createAndLoad(home, home.resolve("solr.xml"));
+ factory = cc.getShardHandlerFactory();
+ assertTrue(factory instanceof HttpShardHandlerFactory);
+ final HttpShardHandlerFactory httpShardHandlerFactory = ((HttpShardHandlerFactory)factory);
+ assertThat(httpShardHandlerFactory.getWhitelistHostChecker().getWhitelistHosts().size(), is(2));
+ assertTrue(httpShardHandlerFactory.getWhitelistHostChecker().getWhitelistHosts().contains("abc:8983"));
+ assertTrue(httpShardHandlerFactory.getWhitelistHostChecker().getWhitelistHosts().contains("def:8984"));
+ } finally {
+ if (factory != null) factory.close();
+ if (cc != null) cc.shutdown();
+ System.clearProperty(SHARDS_WHITELIST);
+ }
+ }
+
+ @Test
+ public void testLiveNodesToHostUrl() throws Exception {
+ Set<String> liveNodes = new HashSet<>(Arrays.asList(new String[]{
+ "1.2.3.4:8983_solr",
+ "1.2.3.4:9000_",
+ "1.2.3.4:9001_solr-2",
+ }));
+ ClusterState cs = new ClusterState(0, liveNodes, new HashMap<>());
+ WhitelistHostChecker checker = new WhitelistHostChecker(null, true);
+ Set<String> hostSet = checker.generateWhitelistFromLiveNodes(cs);
+ assertThat(hostSet.size(), is(3));
+ assertTrue(hostSet.contains("1.2.3.4:8983"));
+ assertTrue(hostSet.contains("1.2.3.4:9000"));
+ assertTrue(hostSet.contains("1.2.3.4:9001"));
+ }
+
+ @Test
+ public void testWhitelistHostCheckerDisabled() throws Exception {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://cde:8983", false);
+ checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"abc-1.com:8983/solr"}));
+
+ try {
+ checker = new WhitelistHostChecker("http://cde:8983", true);
+ checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr"}));
+ fail("Expecting exception");
+ } catch (SolrException se) {
+ assertThat(se.code(), is(SolrException.ErrorCode.FORBIDDEN.code));
+ }
+ }
+
+ @Test
+ public void testWhitelistHostCheckerNoInput() throws Exception {
+ assertNull("Whitelist hosts should be null with null input",
+ new WhitelistHostChecker(null, true).getWhitelistHosts());
+ assertNull("Whitelist hosts should be null with empty input",
+ new WhitelistHostChecker("", true).getWhitelistHosts());
+ }
+
+ @Test
+ public void testWhitelistHostCheckerSingleHost() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983/solr", true);
+ checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr"}));
+ }
+
+ @Test
+ public void testWhitelistHostCheckerMultipleHost() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true);
+ checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr"}));
+ }
+
+ @Test
+ public void testWhitelistHostCheckerMultipleHost2() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true);
+ checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr", "http://abc-2.com:8983/solr"}));
+ }
+
+ @Test
+ public void testWhitelistHostCheckerNoProtocolInParameter() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true);
+ checker.checkWhitelist("abc-1.com:8983/solr", Arrays.asList(new String[]{"abc-1.com:8983/solr"}));
+ }
+
+ @Test
+ public void testWhitelistHostCheckerNonWhitelistedHost1() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true);
+ try {
+ checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-4.com:8983/solr"}));
+ fail("Expected exception");
+ } catch (SolrException e) {
+ assertThat(e.code(), is(SolrException.ErrorCode.FORBIDDEN.code));
+ assertTrue(e.getMessage().contains("not on the shards whitelist"));
+ }
+ }
+
+ @Test
+ public void testWhitelistHostCheckerNonWhitelistedHost2() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true);
+ try {
+ checker.checkWhitelist("http://abc-1.com:8983/solr", Arrays.asList(new String[]{"http://abc-1.com:8983/solr", "http://abc-4.com:8983/solr"}));
+ fail("Expected exception");
+ } catch (SolrException e) {
+ assertThat(e.code(), is(SolrException.ErrorCode.FORBIDDEN.code));
+ assertTrue(e.getMessage().contains("not on the shards whitelist"));
+ }
+ }
+
+ @Test
+ public void testWhitelistHostCheckerNonWhitelistedHostHttps() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true);
+ checker.checkWhitelist("https://abc-1.com:8983/solr", Arrays.asList(new String[]{"https://abc-1.com:8983/solr"}));
+ }
+
+ @Test
+ public void testWhitelistHostCheckerInvalidUrl() {
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983, http://abc-2.com:8983, http://abc-3.com:8983", true);
+ try {
+ checker.checkWhitelist("abc_1", Arrays.asList(new String[]{"abc_1"}));
+ fail("Expected exception");
+ } catch (SolrException e) {
+ assertThat(e.code(), is(SolrException.ErrorCode.BAD_REQUEST.code));
+ assertTrue(e.getMessage().contains("Invalid URL syntax"));
+ }
+ }
+
+ @Test
+ public void testWhitelistHostCheckerCoreSpecific() {
+ // cores are removed completely so it doesn't really matter if they were set in config
+ WhitelistHostChecker checker = new WhitelistHostChecker("http://abc-1.com:8983/solr/core1, http://abc-2.com:8983/solr2/core2", true);
+ checker.checkWhitelist("http://abc-1.com:8983/solr/core2", Arrays.asList(new String[]{"http://abc-1.com:8983/solr/core2"}));
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsUnset() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist(null), nullValue());
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsEmpty() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist(""), nullValue());
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsSingle() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1").size(), is(1));
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1").iterator().next(), equalTo("abc-1.com:8983"));
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsMulti() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1,http://abc-1.com:8984/solr").size(), is(2));
+ assertTrue(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1,http://abc-1.com:8984/solr").contains("abc-1.com:8983"));
+ assertTrue(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr/core1,http://abc-1.com:8984/solr").contains("abc-1.com:8984"));
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsIpv4() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://10.0.0.1:8983/solr/core1,http://127.0.0.1:8984/solr").size(), is(2));
+ assertTrue(WhitelistHostChecker.implGetShardsWhitelist("http://10.0.0.1:8983/solr/core1,http://127.0.0.1:8984/solr").contains("10.0.0.1:8983"));
+ assertTrue(WhitelistHostChecker.implGetShardsWhitelist("http://10.0.0.1:8983/solr/core1,http://127.0.0.1:8984/solr").contains("127.0.0.1:8984"));
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsIpv6() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("http://[2001:abc:abc:0:0:123:456:1234]:8983/solr/core1,http://[::1]:8984/solr").size(), is(2));
+ assertTrue(WhitelistHostChecker.implGetShardsWhitelist("http://[2001:abc:abc:0:0:123:456:1234]:8983/solr/core1,http://[::1]:8984/solr").contains("[2001:abc:abc:0:0:123:456:1234]:8983"));
+ assertTrue(WhitelistHostChecker.implGetShardsWhitelist("http://[2001:abc:abc:0:0:123:456:1234]:8983/solr/core1,http://[::1]:8984/solr").contains("[::1]:8984"));
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsHttps() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("https://abc-1.com:8983/solr/core1").size(), is(1));
+ assertTrue(WhitelistHostChecker.implGetShardsWhitelist("https://abc-1.com:8983/solr/core1").contains("abc-1.com:8983"));
+ }
+
+ @Test
+ public void testGetShardsOfWhitelistedHostsNoProtocol() {
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("abc-1.com:8983/solr"),
+ equalTo(WhitelistHostChecker.implGetShardsWhitelist("http://abc-1.com:8983/solr")));
+ assertThat(WhitelistHostChecker.implGetShardsWhitelist("abc-1.com:8983/solr"),
+ equalTo(WhitelistHostChecker.implGetShardsWhitelist("https://abc-1.com:8983/solr")));
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
index 78760da..e575ebd 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSmileRequest.java
@@ -42,6 +42,7 @@ public class TestSmileRequest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeTests() throws Exception {
+ systemSetPropertySolrDisableShardsWhitelist("true");
JSONTestUtil.failRepeatedKeys = true;
initCore("solrconfig-tlog.xml", "schema_latest.xml");
}
@@ -59,6 +60,7 @@ public class TestSmileRequest extends SolrTestCaseJ4 {
servers.stop();
servers = null;
}
+ systemClearPropertySolrDisableShardsWhitelist();
}
@Test
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
index 6b542a1..fdd07c6 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
@@ -39,6 +39,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
@BeforeClass
public static void beforeTests() throws Exception {
+ systemSetPropertySolrDisableShardsWhitelist("true");
JSONTestUtil.failRepeatedKeys = true;
initCore("solrconfig-tlog.xml","schema_latest.xml");
}
@@ -56,6 +57,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
servers.stop();
servers = null;
}
+ systemClearPropertySolrDisableShardsWhitelist();
}
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index 4ca435c..6838514 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.search.facet;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.tdunning.math.stats.AVLTreeDigest;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,8 +28,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.Random;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import com.tdunning.math.stats.AVLTreeDigest;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.hll.HLL;
@@ -65,14 +65,37 @@ public class TestJsonFacets extends SolrTestCaseHS {
initCore("solrconfig-tlog.xml","schema_latest.xml");
}
+ /**
+ * Start all servers for cluster, initialize shards whitelist and then restart
+ */
public static void initServers() throws Exception {
if (servers == null) {
servers = new SolrInstances(3, "solrconfig-tlog.xml", "schema_latest.xml");
+ // Set the shards whitelist to all shards plus the fake one used for tolerant test
+ System.setProperty(SOLR_TESTS_SHARDS_WHITELIST, servers.getWhitelistString() + ",http://[ff01::114]:33332");
+ systemSetPropertySolrDisableShardsWhitelist("false");
+ restartServers();
}
}
+ /**
+ * Restart all configured servers, i.e. configuration will be re-read
+ */
+ public static void restartServers() {
+ servers.slist.forEach(s -> {
+ try {
+ s.stop();
+ s.start();
+ } catch (Exception e) {
+ fail("Exception during server restart: " + e.getMessage());
+ }
+ });
+ }
+
@AfterClass
public static void afterTests() throws Exception {
+ System.clearProperty(SOLR_TESTS_SHARDS_WHITELIST);
+ systemClearPropertySolrDisableShardsWhitelist();
JSONTestUtil.failRepeatedKeys = false;
FacetFieldProcessorByHashDV.MAXIMUM_STARTING_TABLE_SIZE=origTableSize;
FacetField.FacetMethod.DEFAULT_METHOD = origDefaultFacetMethod;
@@ -1513,7 +1536,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
public void testTolerant() throws Exception {
initServers();
Client client = servers.getClient(random().nextInt());
- client.queryDefaults().set("shards", servers.getShards() + ",[ff01::114]:33332:/ignore_exception");
+ client.queryDefaults().set("shards", servers.getShards() + ",[ff01::114]:33332/ignore_exception");
indexSimple(client);
try {
diff --git a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
index 370ae7e..b42924e 100644
--- a/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
+++ b/solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
@@ -31,6 +31,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
@BeforeClass
public static void beforeTests() throws Exception {
+ systemSetPropertySolrDisableShardsWhitelist("true");
JSONTestUtil.failRepeatedKeys = true;
initCore("solrconfig-tlog.xml","schema_latest.xml");
}
@@ -48,6 +49,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
servers.stop();
servers = null;
}
+ systemClearPropertySolrDisableShardsWhitelist();
}
@Test
diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xml
index 68b15ba..795e352 100644
--- a/solr/server/solr/solr.xml
+++ b/solr/server/solr/solr.xml
@@ -48,6 +48,7 @@
class="HttpShardHandlerFactory">
<int name="socketTimeout">${socketTimeout:600000}</int>
<int name="connTimeout">${connTimeout:60000}</int>
+ <str name="shardsWhitelist">${solr.shardsWhitelist:}</str>
</shardHandlerFactory>
</solr>
diff --git a/solr/solr-ref-guide/src/distributed-requests.adoc b/solr/solr-ref-guide/src/distributed-requests.adoc
index 75f023c..d89f4c6 100644
--- a/solr/solr-ref-guide/src/distributed-requests.adoc
+++ b/solr/solr-ref-guide/src/distributed-requests.adoc
@@ -89,7 +89,7 @@ To configure the standard handler, provide a configuration like this in `solrcon
</requestHandler>
----
-The parameters that can be specified are as follows:
+`HttpShardHandlerFactory` is the only `ShardHandlerFactory` implementation included out of the box with Solr, It accepts the following parameters:
// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed
@@ -105,6 +105,10 @@ The parameters that can be specified are as follows:
|`maxThreadIdleTime` |5 seconds |The amount of time to wait for before threads are scaled back in response to a reduction in load.
|`sizeOfQueue` |-1 |If specified, the thread pool will use a backing queue instead of a direct handoff buffer. High throughput systems will want to configure this to be a direct hand off (with -1). Systems that desire better latency will want to configure a reasonable size of queue to handle variations in requests.
|`fairnessPolicy` |false |Chooses the JVM specifics dealing with fair policy queuing, if enabled distributed searches will be handled in a First in First out fashion at a cost to throughput. If disabled throughput will be favored over latency.
+|`shardsWhitelist` | |If specified, this lists limits what nodes can be requested in the `shards` request parameter. In cloud mode this whitelist is automatically configured to include all live nodes in the cluster. In standalone mode the whitelist defaults to empty (sharding not allowed). If you need to disable this feature for backwards compatibility, you can set the system property `solr.disable.shardsWhitelist=true`. The value of this parameter is a comma separated list of the nodes [...]
+
+NOTE: In cloud mode, if at least one node is included in the shards whitelist, then the live_nodes will no longer be used as source for the list. This means that, if you need to do a cross-cluster request using the `shards` parameter in cloud mode (in addition to regular within-cluster requests), you'll need to add all nodes (local cluster + remote nodes) to the whitelist.
+
|===
[[DistributedRequests-ConfiguringstatsCache_DistributedIDF_]]
diff --git a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc b/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc
index 9b2ca46..4112760 100644
--- a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc
+++ b/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc
@@ -63,6 +63,9 @@ The following components support distributed search:
* The *Stats* component, which returns simple statistics for numeric fields within the DocSet.
* The *Debug* component, which helps with debugging.
+=== Shards Whitelist
+What nodes are allowed in the `shards` parameter is configurable through the `shardsWhitelist` property in `solr.xml`. This whitelist is automatically configured for SolrCloud but needs explicit configuration for master/slave mode. Read more details in <<distributed-requests.adoc#configuring-the-shardhandlerfactory>>.
+
[[DistributedSearchwithIndexSharding-LimitationstoDistributedSearch]]
== Limitations to Distributed Search
diff --git a/solr/solr-ref-guide/src/the-terms-component.adoc b/solr/solr-ref-guide/src/the-terms-component.adoc
index 68e88b0..237d517 100644
--- a/solr/solr-ref-guide/src/the-terms-component.adoc
+++ b/solr/solr-ref-guide/src/the-terms-component.adoc
@@ -308,4 +308,11 @@ The TermsComponent also supports distributed indexes. For the `/terms` request h
[[TheTermsComponent-MoreResources]]
== More Resources
+<<<<<<< HEAD
* {solr-javadocs}/solr-core/org/apache/solr/handler/component/TermsComponent.html[TermsComponent javadoc]
+=======
+`shards.qt`::
+Specifies the request handler Solr uses for requests to shards.
+
+Same as with regular distributed search, the `shards` parameter is subject to a host whitelist that has to be configured in the component init parameters using the configuration key `shardsWhitelist` and the list of hosts as values. In the same way as with distributed search, the whitelist will be populated to all live nodes by default when running in SolrCloud mode. If you need to disable this feature for backwards compatibility, you can set the system property `solr.disable.shardsWhite [...]
+>>>>>>> 6d63958821... SOLR-12770: Make it possible to configure a host whitelist for distributed search
diff --git a/solr/solrj/src/test-files/solrj/solr/solr.xml b/solr/solrj/src/test-files/solrj/solr/solr.xml
index 6eef53f..0e9f3f4 100644
--- a/solr/solrj/src/test-files/solrj/solr/solr.xml
+++ b/solr/solrj/src/test-files/solrj/solr/solr.xml
@@ -30,6 +30,7 @@
<str name="urlScheme">${urlScheme:}</str>
<int name="socketTimeout">${socketTimeout:90000}</int>
<int name="connTimeout">${connTimeout:15000}</int>
+ <str name="shardsWhitelist">${solr.tests.shardsWhitelist:}</str>
</shardHandlerFactory>
<solrcloud>
diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
index 00b7577..c164f6d 100644
--- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
@@ -155,6 +155,18 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
System.clearProperty("hostContext");
}
+ @SuppressWarnings("deprecation")
+ @BeforeClass
+ public static void setSolrDisableShardsWhitelist() throws Exception {
+ systemSetPropertySolrDisableShardsWhitelist("true");
+ }
+
+ @SuppressWarnings("deprecation")
+ @AfterClass
+ public static void clearSolrDisableShardsWhitelist() throws Exception {
+ systemClearPropertySolrDisableShardsWhitelist();
+ }
+
private static String getHostContextSuitableForServletContext() {
String ctx = System.getProperty("hostContext","/solr");
if ("".equals(ctx)) ctx = "/solr";
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java
index 2da0c84..e175566 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java
@@ -64,6 +64,8 @@ import org.slf4j.LoggerFactory;
//@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending","Asserting"})
public class SolrTestCaseHS extends SolrTestCaseJ4 {
+ public static final String SOLR_TESTS_SHARDS_WHITELIST = "solr.tests.shardsWhitelist";
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@SafeVarargs
public static <T> Set<T> set(T... a) {
@@ -468,6 +470,12 @@ public class SolrTestCaseHS extends SolrTestCaseJ4 {
// silly stuff included from solrconfig.snippet.randomindexconfig.xml
System.setProperty("solr.tests.maxBufferedDocs", String.valueOf(100000));
+
+ // If we want to run with whitelist list, this must be explicitly set to true for the test
+ // otherwise we disable the check
+ if (System.getProperty(SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST) == null) {
+ systemSetPropertySolrDisableShardsWhitelist("true");
+ }
jetty.start();
port = jetty.getLocalPort();
@@ -534,6 +542,20 @@ public class SolrTestCaseHS extends SolrTestCaseJ4 {
public String getShards() {
return getShardsParam(slist);
}
+
+ public String getWhitelistString() {
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (SolrInstance instance : slist) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(',');
+ }
+ sb.append( instance.getBaseURL().replace("/solr", ""));
+ }
+ return sb.toString();
+ }
public List<SolrClient> getSolrJs() {
List<SolrClient> solrjs = new ArrayList<>(slist.size());
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index ed6a115..aea5294 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -188,7 +188,10 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
*/
public static final boolean PREFER_POINT_FIELDS = Boolean.getBoolean("solr.tests.preferPointFields");
- private static String coreName = DEFAULT_TEST_CORENAME;
+ @Deprecated // For backwards compatibility only. Please do not use in new tests.
+ public static final String SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST = "solr.disable.shardsWhitelist";
+
+ protected static String coreName = DEFAULT_TEST_CORENAME;
public static int DEFAULT_CONNECTION_TIMEOUT = 60000; // default socket connection timeout in ms
@@ -2482,6 +2485,16 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
System.clearProperty(SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICYFACTORY);
}
+ @Deprecated // For backwards compatibility only. Please do not use in new tests.
+ protected static void systemSetPropertySolrDisableShardsWhitelist(String value) {
+ System.setProperty(SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST, value);
+ }
+
+ @Deprecated // For backwards compatibility only. Please do not use in new tests.
+ protected static void systemClearPropertySolrDisableShardsWhitelist() {
+ System.clearProperty(SYSTEM_PROPERTY_SOLR_DISABLE_SHARDS_WHITELIST);
+ }
+
protected <T> T pickRandom(T... options) {
return options[random().nextInt(options.length)];
}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 76e0d75..d644f50 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -73,6 +73,8 @@ public class MiniSolrCloudCluster {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String SOLR_TESTS_SHARDS_WHITELIST = "solr.tests.shardsWhitelist";
+
public static final String DEFAULT_CLOUD_SOLR_XML = "<solr>\n" +
"\n" +
" <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
@@ -84,6 +86,7 @@ public class MiniSolrCloudCluster {
" <str name=\"urlScheme\">${urlScheme:}</str>\n" +
" <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
" <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
+ " <str name=\"shardsWhitelist\">${"+SOLR_TESTS_SHARDS_WHITELIST+":}</str>\n" +
" </shardHandlerFactory>\n" +
"\n" +
" <solrcloud>\n" +
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index 180cf6e..ccc0c1f 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -85,7 +85,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
/**
* Builder class for a MiniSolrCloudCluster
*/
- protected static class Builder {
+ public static class Builder {
private final int nodeCount;
private final Path baseDir;
@@ -202,7 +202,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
}
/** The cluster */
- protected static MiniSolrCloudCluster cluster;
+ public static MiniSolrCloudCluster cluster;
protected static SolrZkClient zkClient() {
ZkStateReader reader = cluster.getSolrClient().getZkStateReader();