You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2015/11/19 19:14:30 UTC
svn commit: r1715231 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/test/org/apache/solr/core/ solr/test-framework/
solr/test-framework/src/java/org/apache/solr/handler/compo...
Author: cpoerschke
Date: Thu Nov 19 18:14:30 2015
New Revision: 1715231
URL: http://svn.apache.org/viewvc?rev=1715231&view=rev
Log:
SOLR-8298: small preferLocalShards implementation refactor (merge in revision 1715208 from trunk)
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java
lucene/dev/branches/branch_5x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/handler/component/TrackingShardHandlerFactory.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Thu Nov 19 18:14:30 2015
@@ -394,6 +394,8 @@ Other Changes
* SOLR-8283: factor out StrParser from QueryParsing.StrParser and SortSpecParsing[Test]
from QueryParsing[Test] (Christine Poerschke)
+* SOLR-8298: small preferLocalShards implementation refactor (Christine Poerschke)
+
================== 5.3.1 ==================
Bug Fixes
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Thu Nov 19 18:14:30 2015
@@ -114,11 +114,13 @@ public class HttpShardHandler extends Sh
// Not thread safe... don't use in Callable.
// Don't modify the returned URL list.
- private List<String> getURLs(ShardRequest sreq, String shard) {
+ private List<String> getURLs(String shard, String preferredHostAddress) {
List<String> urls = shardToURLs.get(shard);
if (urls == null) {
urls = httpShardHandlerFactory.makeURLList(shard);
- preferCurrentHostForDistributedReq(sreq, urls);
+ if (preferredHostAddress != null && urls.size() > 1) {
+ preferCurrentHostForDistributedReq(preferredHostAddress, urls);
+ }
shardToURLs.put(shard, urls);
}
return urls;
@@ -131,27 +133,7 @@ public class HttpShardHandler extends Sh
* If all nodes prefer local-cores then a bad/heavily-loaded node will receive less requests from healthy nodes.
* This will help prevent a distributed deadlock or timeouts in all the healthy nodes due to one bad node.
*/
- private void preferCurrentHostForDistributedReq(final ShardRequest sreq, final List<String> urls) {
- if (sreq == null || sreq.rb == null || sreq.rb.req == null || urls == null || urls.size() <= 1)
- return;
-
- SolrQueryRequest req = sreq.rb.req;
-
- // determine if we should apply the local preference
- if (!req.getOriginalParams().getBool(CommonParams.PREFER_LOCAL_SHARDS, false))
- return;
-
- // Get this node's base URL from ZK
- SolrCore core = req.getCore();
- ZkController zkController = (core != null) ? core.getCoreDescriptor().getCoreContainer().getZkController() : null;
- String currentHostAddress = (zkController != null) ? zkController.getBaseUrl() : null;
- if (currentHostAddress == null) {
- log.debug("Couldn't determine current host address to prefer local shards " +
- "because either core is null? {} or there is no ZkController? {}",
- Boolean.valueOf(core == null), Boolean.valueOf(zkController == null));
- return;
- }
-
+ private void preferCurrentHostForDistributedReq(final String currentHostAddress, final List<String> urls) {
if (log.isDebugEnabled())
log.debug("Trying to prefer local shard on {} among the urls: {}",
currentHostAddress, Arrays.toString(urls.toArray()));
@@ -174,9 +156,9 @@ public class HttpShardHandler extends Sh
}
@Override
- public void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) {
+ public void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params, String preferredHostAddress) {
// do this outside of the callable for thread safety reasons
- final List<String> urls = getURLs(sreq, shard);
+ final List<String> urls = getURLs(shard, preferredHostAddress);
Callable<ShardResponse> task = new Callable<ShardResponse>() {
@Override
@@ -335,6 +317,12 @@ public class HttpShardHandler extends Sh
CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
ZkController zkController = coreDescriptor.getCoreContainer().getZkController();
+ if (params.getBool(CommonParams.PREFER_LOCAL_SHARDS, false)) {
+ rb.preferredHostAddress = (zkController != null) ? zkController.getBaseUrl() : null;
+ if (rb.preferredHostAddress == null) {
+ log.warn("Couldn't determine current host address to prefer local shards");
+ }
+ }
if (shards != null) {
List<String> lst = StrUtils.splitSmart(shards, ",", true);
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java Thu Nov 19 18:14:30 2015
@@ -136,6 +136,7 @@ public class ResponseBuilder
public int shards_start = -1;
public List<ShardRequest> outgoing; // requests to be sent
public List<ShardRequest> finished; // requests that have received responses from all shards
+ public String preferredHostAddress = null;
public String shortCircuitedURL;
public int getShardNum(String shard) {
@@ -147,7 +148,6 @@ public class ResponseBuilder
public void addRequest(SearchComponent me, ShardRequest sreq) {
outgoing.add(sreq);
- sreq.rb = this;
if ((sreq.purpose & ShardRequest.PURPOSE_PRIVATE) == 0) {
// if this isn't a private request, let other components modify it.
for (SearchComponent component : components) {
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java Thu Nov 19 18:14:30 2015
@@ -370,7 +370,7 @@ public class SearchHandler extends Reque
params.remove(CommonParams.QT);
}
}
- shardHandler1.submit(sreq, shard, params);
+ shardHandler1.submit(sreq, shard, params, rb.preferredHostAddress);
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardHandler.java Thu Nov 19 18:14:30 2015
@@ -21,7 +21,10 @@ import org.apache.solr.common.params.Mod
public abstract class ShardHandler {
public abstract void prepDistributed(ResponseBuilder rb);
- public abstract void submit(ShardRequest sreq, String shard, ModifiableSolrParams params) ;
+ public void submit(ShardRequest sreq, String shard, ModifiableSolrParams params) {
+ submit(sreq, shard, params, null);
+ }
+ public abstract void submit(ShardRequest sreq, String shard, ModifiableSolrParams params, String preferredHostAddress);
public abstract ShardResponse takeCompletedIncludingErrors();
public abstract ShardResponse takeCompletedOrError();
public abstract void cancelAll();
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java Thu Nov 19 18:14:30 2015
@@ -49,7 +49,6 @@ public class ShardRequest {
public ModifiableSolrParams params;
- public ResponseBuilder rb;
/** list of responses... filled out by framework */
public List<ShardResponse> responses = new ArrayList<>();
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java Thu Nov 19 18:14:30 2015
@@ -43,7 +43,7 @@ public class MockShardHandlerFactory ext
@Override
public void submit(ShardRequest sreq, String shard,
- ModifiableSolrParams params) {}
+ ModifiableSolrParams params, String preferredHostAddress) {}
@Override
public ShardResponse takeCompletedIncludingErrors() {
Modified: lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/handler/component/TrackingShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/handler/component/TrackingShardHandlerFactory.java?rev=1715231&r1=1715230&r2=1715231&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/handler/component/TrackingShardHandlerFactory.java (original)
+++ lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/handler/component/TrackingShardHandlerFactory.java Thu Nov 19 18:14:30 2015
@@ -92,7 +92,7 @@ public class TrackingShardHandlerFactory
}
@Override
- public void submit(ShardRequest sreq, String shard, ModifiableSolrParams params) {
+ public void submit(ShardRequest sreq, String shard, ModifiableSolrParams params, String preferredHostAddress) {
synchronized (TrackingShardHandlerFactory.this) {
if (isTracking()) {
queue.offer(new ShardRequestAndParams(sreq, shard, params));