You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/08/05 00:07:32 UTC
svn commit: r1369487 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/solrj/ solr/solrj/src/java/org/apache/solr/client/solrj/impl/
solr/solrj/src/java/org/apache/solr/client/solrj/request/
solr/solrj/src/java/org/apache/solr/common/cloud/
Author: markrmiller
Date: Sat Aug 4 22:07:32 2012
New Revision: 1369487
URL: http://svn.apache.org/viewvc?rev=1369487&view=rev
Log:
SOLR-3710: Change CloudSolrServer so that update requests are only sent to leaders by default.
Added:
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/IsUpdateRequest.java
- copied unchanged from r1369484, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/IsUpdateRequest.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1369487&r1=1369486&r2=1369487&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sat Aug 4 22:07:32 2012
@@ -113,6 +113,9 @@ Optimizations
* SOLR-3709: Cache the url list created from the ClusterState in CloudSolrServer on each
request. (Mark Miller, yonik)
+
+* SOLR-3710: Change CloudSolrServer so that update requests are only sent to leaders by
+ default. (Mark Miller)
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1369487&r1=1369486&r2=1369487&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Sat Aug 4 22:07:32 2012
@@ -32,6 +32,7 @@ import org.apache.http.client.HttpClient
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.IsUpdateRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
@@ -64,7 +65,11 @@ public class CloudSolrServer extends Sol
// since the state shouldn't change often, should be very cheap reads
private volatile List<String> urlList;
+ private volatile List<String> leaderUrlList;
private volatile int lastClusterStateHashCode;
+
+ private final boolean updatesToLeaders;
+
/**
* @param zkHost The client endpoint of the zookeeper quorum containing the cloud state,
* in the form HOST:PORT.
@@ -73,6 +78,7 @@ public class CloudSolrServer extends Sol
this.zkHost = zkHost;
this.myClient = HttpClientUtil.createClient(null);
this.lbServer = new LBHttpSolrServer(myClient);
+ this.updatesToLeaders = true;
}
/**
@@ -83,6 +89,19 @@ public class CloudSolrServer extends Sol
public CloudSolrServer(String zkHost, LBHttpSolrServer lbServer) {
this.zkHost = zkHost;
this.lbServer = lbServer;
+ this.updatesToLeaders = true;
+ }
+
+ /**
+ * @param zkHost The client endpoint of the zookeeper quorum containing the cloud state,
+ * in the form HOST:PORT.
+ * @param lbServer LBHttpSolrServer instance for requests.
+ * @param updatesToLeaders sends updates only to leaders - defaults to true
+ */
+ public CloudSolrServer(String zkHost, LBHttpSolrServer lbServer, boolean updatesToLeaders) {
+ this.zkHost = zkHost;
+ this.lbServer = lbServer;
+ this.updatesToLeaders = updatesToLeaders;
}
public ZkStateReader getZkStateReader() {
@@ -144,6 +163,11 @@ public class CloudSolrServer extends Sol
// TODO: if you can hash here, you could favor the shard leader
ClusterState clusterState = zkStateReader.getClusterState();
+ boolean sendToLeaders = false;
+
+ if (request instanceof IsUpdateRequest && updatesToLeaders) {
+ sendToLeaders = true;
+ }
SolrParams reqParams = request.getParams();
if (reqParams == null) {
@@ -158,6 +182,9 @@ public class CloudSolrServer extends Sol
// Extract each comma separated collection name and store in a List.
List<String> collectionList = StrUtils.splitSmart(collection, ",", true);
+ // TODO: not a big deal because of the caching, but we could avoid looking at every shard
+ // when getting leaders if we tweaked some things
+
// Retrieve slices from the cloud state and, for each collection specified,
// add it to the Map of slices.
Map<String,Slice> slices = new HashMap<String,Slice>();
@@ -168,11 +195,7 @@ public class CloudSolrServer extends Sol
Set<String> liveNodes = clusterState.getLiveNodes();
- // IDEA: have versions on various things... like a global clusterState version
- // or shardAddressVersion (which only changes when the shards change)
- // to allow caching.
-
- if (clusterState.hashCode() != this.lastClusterStateHashCode) {
+ if (sendToLeaders && leaderUrlList == null || !sendToLeaders && urlList == null || clusterState.hashCode() != this.lastClusterStateHashCode) {
// build a map of unique nodes
// TODO: allow filtering by group, role, etc
@@ -185,19 +208,32 @@ public class CloudSolrServer extends Sol
if (!liveNodes.contains(coreNodeProps.getNodeName())
|| !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue;
if (nodes.put(node, nodeProps) == null) {
- String url = coreNodeProps.getCoreUrl();
- urlList.add(url);
+ if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) {
+ String url = coreNodeProps.getCoreUrl();
+ urlList.add(url);
+ }
}
}
}
- this.urlList = urlList;
+ if (sendToLeaders) {
+ this.leaderUrlList = urlList;
+ } else {
+ this.urlList = urlList;
+ }
this.lastClusterStateHashCode = clusterState.hashCode();
}
-
- Collections.shuffle(urlList, rand);
+ List<String> theUrlList;
+ if (sendToLeaders) {
+ theUrlList = new ArrayList<String>(leaderUrlList.size());
+ theUrlList.addAll(leaderUrlList);
+ } else {
+ theUrlList = new ArrayList<String>(urlList.size());
+ theUrlList.addAll(urlList);
+ }
+ Collections.shuffle(theUrlList, rand);
//System.out.println("########################## MAKING REQUEST TO " + urlList);
- LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, urlList);
+ LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, theUrlList);
LBHttpSolrServer.Rsp rsp = lbServer.request(req);
return rsp.getResponse();
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java?rev=1369487&r1=1369486&r2=1369487&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java Sat Aug 4 22:07:32 2012
@@ -30,7 +30,7 @@ import java.io.IOException;
*
*
**/
-public abstract class AbstractUpdateRequest extends SolrRequest {
+public abstract class AbstractUpdateRequest extends SolrRequest implements IsUpdateRequest {
protected ModifiableSolrParams params;
protected int commitWithin = -1;
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java?rev=1369487&r1=1369486&r2=1369487&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java Sat Aug 4 22:07:32 2012
@@ -34,7 +34,7 @@ import org.apache.solr.common.util.Conte
*
* @since solr 1.3
*/
-public class DirectXmlRequest extends SolrRequest
+public class DirectXmlRequest extends SolrRequest implements IsUpdateRequest
{
final String xml;
private SolrParams params;
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java?rev=1369487&r1=1369486&r2=1369487&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java Sat Aug 4 22:07:32 2012
@@ -74,5 +74,9 @@ public class ZkCoreNodeProps {
return nodeProps;
}
+ public boolean isLeader() {
+ return nodeProps.containsKey(ZkStateReader.LEADER_PROP);
+ }
+
}