You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2013/03/07 06:58:43 UTC
svn commit: r1453691 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/cloud/
solr/core/src/java/org/apache/solr/handler/admin/
solr/core/src/java/org/apache/solr/servlet/
solr/core/src/test/org/apache/solr/clo...
Author: markrmiller
Date: Thu Mar 7 05:58:43 2013
New Revision: 1453691
URL: http://svn.apache.org/r1453691
Log:
SOLR-4497: Collection Aliasing.
Added:
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
- copied unchanged from r1453687, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
- copied unchanged from r1453687, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Thu Mar 7 05:58:43 2013
@@ -200,6 +200,8 @@ Bug Fixes
dynamic field references that aren't string-equal to the name of
the referenced dynamic field. (Steve Rowe)
+* SOLR-4497: Collection Aliasing. (Mark Miller)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Thu Mar 7 05:58:43 2013
@@ -19,6 +19,7 @@ package org.apache.solr.cloud;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -27,6 +28,7 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.cloud.DistributedQueue.QueueEvent;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClosableThread;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -63,10 +65,12 @@ public class OverseerCollectionProcessor
public static final String RELOADCOLLECTION = "reloadcollection";
+ public static final String CREATEALIAS = "createalias";
+
+ public static final String DELETEALIAS = "deletealias";
+
// TODO: use from Overseer?
private static final String QUEUE_OPERATION = "operation";
-
-
private static Logger log = LoggerFactory
.getLogger(OverseerCollectionProcessor.class);
@@ -162,6 +166,10 @@ public class OverseerCollectionProcessor
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.RELOAD.toString());
collectionCmd(zkStateReader.getClusterState(), message, params);
+ } else if (CREATEALIAS.equals(operation)) {
+ createAlias(zkStateReader.getAliases(), message);
+ } else if (DELETEALIAS.equals(operation)) {
+ deleteAlias(zkStateReader.getAliases(), message);
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknow the operation:"
+ operation);
@@ -182,15 +190,116 @@ public class OverseerCollectionProcessor
}
}
} while (srsp != null);
- } catch (SolrException ex) {
+ } catch (Exception ex) {
SolrException.log(log, "Collection " + operation + " of " + operation
- + " failed");
+ + " failed", ex);
results.add("Operation " + operation + " caused exception:", ex);
} finally {
return new OverseerSolrResponse(results);
}
}
+ private void createAlias(Aliases aliases, ZkNodeProps message) {
+ String aliasName = message.getStr("name");
+ String collections = message.getStr("collections");
+
+ Map<String,Map<String,String>> newAliasesMap = new HashMap<String,Map<String,String>>();
+ Map<String,String> newCollectionAliasesMap = new HashMap<String,String>();
+ Map<String,String> prevColAliases = aliases.getCollectionAliasMap();
+ if (prevColAliases != null) {
+ newCollectionAliasesMap.putAll(prevColAliases);
+ }
+ newCollectionAliasesMap.put(aliasName, collections);
+ newAliasesMap.put("collection", newCollectionAliasesMap);
+ Aliases newAliases = new Aliases(newAliasesMap);
+ byte[] jsonBytes = null;
+ if (newAliases.collectionAliasSize() > 0) { // only sub map right now
+ jsonBytes = ZkStateReader.toJSON(newAliases.getAliasMap());
+ }
+ try {
+ zkStateReader.getZkClient().setData(ZkStateReader.ALIASES,
+ jsonBytes, true);
+
+ checkForAlias(aliasName, collections);
+ // some fudge for other nodes
+ Thread.sleep(100);
+ } catch (KeeperException e) {
+ log.error("", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ log.warn("", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+
+ }
+
+ private void checkForAlias(String name, String value) {
+
+ long now = System.currentTimeMillis();
+ long timeout = now + 30000;
+ boolean success = false;
+ Aliases aliases = null;
+ while (System.currentTimeMillis() < timeout) {
+ aliases = zkStateReader.getAliases();
+ String collections = aliases.getCollectionAlias(name);
+ if (collections != null && collections.equals(value)) {
+ success = true;
+ break;
+ }
+ }
+ if (!success) {
+ log.warn("Timeout waiting to be notified of Alias change...");
+ }
+ }
+
+ private void checkForAliasAbsence(String name) {
+
+ long now = System.currentTimeMillis();
+ long timeout = now + 30000;
+ boolean success = false;
+ Aliases aliases = null;
+ while (System.currentTimeMillis() < timeout) {
+ aliases = zkStateReader.getAliases();
+ String collections = aliases.getCollectionAlias(name);
+ if (collections == null) {
+ success = true;
+ break;
+ }
+ }
+ if (!success) {
+ log.warn("Timeout waiting to be notified of Alias change...");
+ }
+ }
+
+ private void deleteAlias(Aliases aliases, ZkNodeProps message) {
+ String aliasName = message.getStr("name");
+
+ Map<String,Map<String,String>> newAliasesMap = new HashMap<String,Map<String,String>>();
+ Map<String,String> newCollectionAliasesMap = new HashMap<String,String>();
+ newCollectionAliasesMap.putAll(aliases.getCollectionAliasMap());
+ newCollectionAliasesMap.remove(aliasName);
+ newAliasesMap.put("collection", newCollectionAliasesMap);
+ Aliases newAliases = new Aliases(newAliasesMap);
+ byte[] jsonBytes = null;
+ if (newAliases.collectionAliasSize() > 0) { // only sub map right now
+ jsonBytes = ZkStateReader.toJSON(newAliases.getAliasMap());
+ }
+ try {
+ zkStateReader.getZkClient().setData(ZkStateReader.ALIASES,
+ jsonBytes, true);
+ checkForAliasAbsence(aliasName);
+ // some fudge for other nodes
+ Thread.sleep(100);
+ } catch (KeeperException e) {
+ log.error("", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ log.warn("", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+
+ }
+
private boolean createCollection(ClusterState clusterState, ZkNodeProps message) {
String collectionName = message.getStr("name");
if (clusterState.getCollections().contains(collectionName)) {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Thu Mar 7 05:58:43 2013
@@ -120,7 +120,14 @@ public class CollectionsHandler extends
this.handleSyncShardAction(req, rsp);
break;
}
-
+ case CREATEALIAS: {
+ this.handleCreateAliasAction(req, rsp);
+ break;
+ }
+ case DELETEALIAS: {
+ this.handleDeleteAliasAction(req, rsp);
+ break;
+ }
default: {
throw new RuntimeException("Unknown action: " + action);
}
@@ -187,7 +194,30 @@ public class CollectionsHandler extends
reqSyncShard.setCoreName(nodeProps.getCoreName());
server.request(reqSyncShard);
}
-
+
+ private void handleCreateAliasAction(SolrQueryRequest req,
+ SolrQueryResponse rsp) throws Exception {
+ log.info("Create alias action : " + req.getParamString());
+ String name = req.getParams().required().get("name");
+ String collections = req.getParams().required().get("collections");
+
+ ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
+ OverseerCollectionProcessor.CREATEALIAS, "name", name, "collections",
+ collections);
+
+ handleResponse(OverseerCollectionProcessor.CREATEALIAS, m, rsp);
+ }
+
+ private void handleDeleteAliasAction(SolrQueryRequest req,
+ SolrQueryResponse rsp) throws Exception {
+ log.info("Delete alias action : " + req.getParamString());
+ String name = req.getParams().required().get("name");
+
+ ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
+ OverseerCollectionProcessor.DELETEALIAS, "name", name);
+
+ handleResponse(OverseerCollectionProcessor.CREATEALIAS, m, rsp);
+ }
private void handleDeleteAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
log.info("Deleting Collection : " + req.getParamString());
@@ -200,7 +230,6 @@ public class CollectionsHandler extends
handleResponse(OverseerCollectionProcessor.DELETECOLLECTION, m, rsp);
}
-
// very simple currently, you can pass a template collection, and the new collection is created on
// every node the template collection is on
// there is a lot more to add - you should also be able to create with an explicit server list
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Thu Mar 7 05:58:43 2013
@@ -17,8 +17,6 @@
package org.apache.solr.servlet;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -31,6 +29,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -42,7 +41,6 @@ import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
-import javax.servlet.ServletOutputStream;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
@@ -50,6 +48,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -58,10 +57,12 @@ import org.apache.solr.common.cloud.ZkNo
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
@@ -134,9 +135,13 @@ public class SolrDispatchFilter implemen
cores = null;
}
}
-
+
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
+ doFilter(request, response, chain, false);
+ }
+
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain, boolean retry) throws IOException, ServletException {
if( abortErrorMessage != null ) {
((HttpServletResponse)response).sendError( 500, abortErrorMessage );
return;
@@ -149,12 +154,14 @@ public class SolrDispatchFilter implemen
CoreContainer cores = this.cores;
SolrCore core = null;
SolrQueryRequest solrReq = null;
+ Aliases aliases = null;
if( request instanceof HttpServletRequest) {
HttpServletRequest req = (HttpServletRequest)request;
HttpServletResponse resp = (HttpServletResponse)response;
SolrRequestHandler handler = null;
String corename = "";
+ String origCorename = null;
try {
// put the core container in request attribute
req.setAttribute("org.apache.solr.CoreContainer", cores);
@@ -185,6 +192,8 @@ public class SolrDispatchFilter implemen
handleAdminRequest(req, response, handler, solrReq);
return;
}
+ boolean usingAliases = false;
+ List<String> collectionsList = null;
// Check for the core admin collections url
if( path.equals( "/admin/collections" ) ) {
handler = cores.getCollectionsHandler();
@@ -198,7 +207,24 @@ public class SolrDispatchFilter implemen
if( idx > 1 ) {
// try to get the corename as a request parameter first
corename = path.substring( 1, idx );
+
+ // look at aliases
+ if (cores.isZooKeeperAware()) {
+ origCorename = corename;
+ ZkStateReader reader = cores.getZkController().getZkStateReader();
+ aliases = reader.getAliases();
+ if (aliases != null && aliases.collectionAliasSize() > 0) {
+ usingAliases = true;
+ String alias = aliases.getCollectionAlias(corename);
+ if (alias != null) {
+ collectionsList = StrUtils.splitSmart(alias, ",", true);
+ corename = collectionsList.get(0);
+ }
+ }
+ }
+
core = cores.getCore(corename);
+
if (core != null) {
path = path.substring( idx );
}
@@ -221,11 +247,19 @@ public class SolrDispatchFilter implemen
// if we couldn't find it locally, look on other nodes
if (core == null && idx > 0) {
- String coreUrl = getRemotCoreUrl(cores, corename);
+ String coreUrl = getRemotCoreUrl(cores, corename, origCorename);
if (coreUrl != null) {
path = path.substring( idx );
remoteQuery(coreUrl + path, req, solrReq, resp);
return;
+ } else {
+ if (!retry) {
+ // we couldn't find a core to work with, try reloading aliases
+ ZkStateReader reader = cores.getZkController()
+ .getZkStateReader();
+ reader.updateAliases();
+ doFilter(request, response, chain, true);
+ }
}
}
@@ -289,6 +323,10 @@ public class SolrDispatchFilter implemen
solrReq = parser.parse( core, path, req );
}
+ if (usingAliases) {
+ processAliases(solrReq, aliases, collectionsList);
+ }
+
final Method reqMethod = Method.getMethod(req.getMethod());
HttpCacheHeaderUtil.setCacheControlHeader(config, resp, reqMethod);
// unless we have been explicitly told not to, do cache validation
@@ -341,9 +379,44 @@ public class SolrDispatchFilter implemen
chain.doFilter(request, response);
}
+ private void processAliases(SolrQueryRequest solrReq, Aliases aliases,
+ List<String> collectionsList) {
+ String collection = solrReq.getParams().get("collection");
+ if (collection != null) {
+ collectionsList = StrUtils.splitSmart(collection, ",", true);
+ }
+ if (collectionsList != null) {
+ Set<String> newCollectionsList = new HashSet<String>(
+ collectionsList.size());
+ for (String col : collectionsList) {
+ String al = aliases.getCollectionAlias(col);
+ if (al != null) {
+ List<String> aliasList = StrUtils.splitSmart(al, ",", true);
+ newCollectionsList.addAll(aliasList);
+ } else {
+ newCollectionsList.add(col);
+ }
+ }
+ if (newCollectionsList.size() > 0) {
+ StringBuilder collectionString = new StringBuilder();
+ Iterator<String> it = newCollectionsList.iterator();
+ int sz = newCollectionsList.size();
+ for (int i = 0; i < sz; i++) {
+ collectionString.append(it.next());
+ if (i < newCollectionsList.size() - 1) {
+ collectionString.append(",");
+ }
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams(
+ solrReq.getParams());
+ params.set("collection", collectionString.toString());
+ solrReq.setParams(params);
+ }
+ }
+ }
+
private void remoteQuery(String coreUrl, HttpServletRequest req,
SolrQueryRequest solrReq, HttpServletResponse resp) throws IOException {
-
try {
String urlstr = coreUrl;
@@ -409,7 +482,7 @@ public class SolrDispatchFilter implemen
}
- private String getRemotCoreUrl(CoreContainer cores, String collectionName) {
+ private String getRemotCoreUrl(CoreContainer cores, String collectionName, String origCorename) {
ClusterState clusterState = cores.getZkController().getClusterState();
Collection<Slice> slices = clusterState.getSlices(collectionName);
boolean byCoreName = false;
@@ -444,9 +517,14 @@ public class SolrDispatchFilter implemen
// don't count a local core
continue;
}
- String coreUrl = coreNodeProps.getCoreUrl();
- if (coreUrl.endsWith("/")) {
- coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
+ String coreUrl;
+ if (origCorename != null) {
+ coreUrl = coreNodeProps.getBaseUrl() + "/" + origCorename;
+ } else {
+ coreUrl = coreNodeProps.getCoreUrl();
+ if (coreUrl.endsWith("/")) {
+ coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
+ }
}
return coreUrl;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Thu Mar 7 05:58:43 2013
@@ -330,12 +330,5 @@ public class ChaosMonkeyNothingIsSafeTes
SolrInputDocument doc = getDoc(fields);
indexDoc(doc);
}
-
- SolrInputDocument getDoc(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- addFields(doc, "rnd_b", true);
- return doc;
- }
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Thu Mar 7 05:58:43 2013
@@ -44,8 +44,6 @@ import javax.management.ObjectName;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util._TestUtil;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -149,10 +147,7 @@ public class CollectionsAPIDistributedZk
private void testNodesUsedByCreate() throws Exception {
// we can use this client because we just want base url
- final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(0)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
+ final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
@@ -297,10 +292,7 @@ public class CollectionsAPIDistributedZk
request.setPath("/admin/collections");
// we can use this client because we just want base url
- final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(0)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
+ final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
createNewSolrServer("", baseUrl).request(request);
@@ -409,38 +401,6 @@ public class CollectionsAPIDistributedZk
checkNoTwoShardsUseTheSameIndexDir();
}
-
- protected void createCollection(Map<String,List<Integer>> collectionInfos,
- String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("action", CollectionAction.CREATE.toString());
-
- params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
- params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
- params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
- if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
-
- int clientIndex = random().nextInt(2);
- List<Integer> list = new ArrayList<Integer>();
- list.add(numShards);
- list.add(numReplicas);
- collectionInfos.put(collectionName, list);
- params.set("name", collectionName);
- SolrRequest request = new QueryRequest(params);
- request.setPath("/admin/collections");
-
- if (client == null) {
- final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
- 0,
- ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
- - DEFAULT_COLLECTION.length() - 1);
-
- createNewSolrServer("", baseUrl).request(request);
- } else {
- client.request(request);
- }
- }
-
private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
@@ -549,60 +509,6 @@ public class CollectionsAPIDistributedZk
fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
}
- private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
- ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-
- int expectedSlices = numShardsNumReplicaList.get(0);
- // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
- int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
- int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
-
- Map<String,DocCollection> collections = clusterState
- .getCollectionStates();
- if (collections.containsKey(collectionName)) {
- Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
- // did we find expectedSlices slices/shards?
- if (slices.size() != expectedSlices) {
- return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
- }
- int totalShards = 0;
- for (String sliceName : slices.keySet()) {
- for (Replica replica : slices.get(sliceName).getReplicas()) {
- if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
- return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
- }
- }
- totalShards += slices.get(sliceName).getReplicas().size();
- }
- if (totalShards != expectedTotalShards) {
- return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
- }
- return null;
- } else {
- return "Could not find new collection " + collectionName;
- }
- }
-
- private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
- throws Exception {
- // check for an expectedSlices new collection - we poll the state
- long timeoutAt = System.currentTimeMillis() + 120000;
- boolean success = false;
- String checkResult = "Didnt get to perform a single check";
- while (System.currentTimeMillis() < timeoutAt) {
- checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
- if (checkResult == null) {
- success = true;
- break;
- }
- Thread.sleep(500);
- }
- if (!success) {
- super.printLayout();
- fail(checkResult);
- }
- }
-
private void checkCollectionIsNotCreated(String collectionName)
throws Exception {
// TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
@@ -678,42 +584,6 @@ public class CollectionsAPIDistributedZk
}
}
-
- protected SolrInputDocument getDoc(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- return doc;
- }
-
- protected SolrServer createNewSolrServer(String collection, String baseUrl) {
- try {
- // setup the server...
- HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
- s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
- s.setDefaultMaxConnectionsPerHost(100);
- s.setMaxTotalConnections(100);
- return s;
- }
- catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
-
- volatile CloudSolrServer commondCloudSolrServer;
- private CloudSolrServer getCommonCloudSolrServer() {
- if (commondCloudSolrServer == null) {
- synchronized(this) {
- try {
- commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
- commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
- commondCloudSolrServer.connect();
- } catch (MalformedURLException e) {
- throw new RuntimeException(e);
- }
- }
- }
- return commondCloudSolrServer;
- }
@Override
protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
@@ -731,9 +601,6 @@ public class CollectionsAPIDistributedZk
@Override
public void tearDown() throws Exception {
super.tearDown();
- if (commondCloudSolrServer != null) {
- commondCloudSolrServer.shutdown();
- }
System.clearProperty("numShards");
System.clearProperty("zkHost");
System.clearProperty("solr.xml.persist");
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Thu Mar 7 05:58:43 2013
@@ -19,10 +19,13 @@ package org.apache.solr.client.solrj.imp
import java.io.IOException;
import java.net.MalformedURLException;
+import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -36,6 +39,8 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.client.solrj.request.IsUpdateRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -183,13 +188,13 @@ public class CloudSolrServer extends Sol
reqParams = new ModifiableSolrParams();
}
List<String> theUrlList = new ArrayList<String>();
- if (request.getPath().equals("/admin/collections")) {
+ if (request.getPath().equals("/admin/collections") || request.getPath().equals("/admin/cores")) {
Set<String> liveNodes = clusterState.getLiveNodes();
for (String liveNode : liveNodes) {
int splitPointBetweenHostPortAndContext = liveNode.indexOf("_");
theUrlList.add("http://"
+ liveNode.substring(0, splitPointBetweenHostPortAndContext) + "/"
- + liveNode.substring(splitPointBetweenHostPortAndContext + 1));
+ + URLDecoder.decode(liveNode, "UTF-8").substring(splitPointBetweenHostPortAndContext + 1));
}
} else {
String collection = reqParams.get("collection", defaultCollection);
@@ -199,9 +204,21 @@ public class CloudSolrServer extends Sol
"No collection param specified on request and no default collection has been set.");
}
- // Extract each comma separated collection name and store in a List.
- List<String> collectionList = StrUtils.splitSmart(collection, ",", true);
+ Set<String> collectionsList = getCollectionList(clusterState, collection);
+ if (collectionsList.size() == 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
+ }
+ collection = collectionsList.iterator().next();
+ StringBuilder collectionString = new StringBuilder();
+ Iterator<String> it = collectionsList.iterator();
+ for (int i = 0; i < collectionsList.size(); i++) {
+ String col = it.next();
+ collectionString.append(col);
+ if (i < collectionsList.size() - 1) {
+ collectionString.append(",");
+ }
+ }
// TODO: not a big deal because of the caching, but we could avoid looking
// at every shard
// when getting leaders if we tweaked some things
@@ -210,9 +227,12 @@ public class CloudSolrServer extends Sol
// specified,
// add it to the Map of slices.
Map<String,Slice> slices = new HashMap<String,Slice>();
- for (String collectionName : collectionList) {
- ClientUtils.addSlices(slices, collectionName,
- clusterState.getSlices(collectionName), true);
+ for (String collectionName : collectionsList) {
+ Collection<Slice> colSlices = clusterState.getSlices(collectionName);
+ if (colSlices == null) {
+ throw new SolrServerException("Could not find collection:" + collectionName);
+ }
+ ClientUtils.addSlices(slices, collectionName, colSlices, true);
}
Set<String> liveNodes = clusterState.getLiveNodes();
@@ -287,6 +307,30 @@ public class CloudSolrServer extends Sol
return rsp.getResponse();
}
+ private Set<String> getCollectionList(ClusterState clusterState,
+ String collection) {
+ // Extract each comma separated collection name and store in a List.
+ List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
+ Set<String> collectionsList = new HashSet<String>();
+ // validate collections
+ for (String collectionName : rawCollectionsList) {
+ if (!clusterState.getCollections().contains(collectionName)) {
+ Aliases aliases = zkStateReader.getAliases();
+ String alias = aliases.getCollectionAlias(collectionName);
+ if (alias != null) {
+ List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
+ collectionsList.addAll(aliasList);
+ continue;
+ }
+
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
+ }
+
+ collectionsList.add(collectionName);
+ }
+ return collectionsList;
+ }
+
@Override
public void shutdown() {
if (zkStateReader != null) {
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java Thu Mar 7 05:58:43 2013
@@ -17,13 +17,11 @@ package org.apache.solr.common.cloud;
* limitations under the License.
*/
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -31,7 +29,6 @@ import java.util.Set;
import org.apache.noggit.JSONWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.DocRouter.Range;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
@@ -229,6 +226,15 @@ public class ClusterState implements JSO
// System.out.println("######## ClusterState.load result:" + collections);
return new ClusterState(version, liveNodes, collections);
}
+
+ public static Aliases load(byte[] bytes) {
+ if (bytes == null || bytes.length == 0) {
+ return new Aliases();
+ }
+ Map<String,Map<String,String>> aliasMap = (Map<String,Map<String,String>>) ZkStateReader.fromJSON(bytes);
+
+ return new Aliases(aliasMap);
+ }
private static DocCollection collectionFromObjects(String name, Map<String,Object> objs) {
Map<String,Object> props;
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Thu Mar 7 05:58:43 2013
@@ -62,6 +62,7 @@ public class ZkStateReader {
public static final String COLLECTIONS_ZKNODE = "/collections";
public static final String LIVE_NODES_ZKNODE = "/live_nodes";
+ public static final String ALIASES = "/aliases.json";
public static final String CLUSTER_STATE = "/clusterstate.json";
public static final String RECOVERING = "recovering";
@@ -129,7 +130,9 @@ public class ZkStateReader {
private boolean closeClient = false;
private ZkCmdExecutor cmdExecutor;
-
+
+ private Aliases aliases = new Aliases();
+
public ZkStateReader(SolrZkClient zkClient) {
this.zkClient = zkClient;
initZkCmdExecutor(zkClient.getZkClientTimeout());
@@ -177,12 +180,17 @@ public class ZkStateReader {
updateClusterState(true, true);
}
+ public Aliases getAliases() {
+ return aliases;
+ }
+
public synchronized void createClusterStateWatchersAndUpdate() throws KeeperException,
InterruptedException {
// We need to fetch the current cluster state and the set of live nodes
synchronized (getUpdateLock()) {
cmdExecutor.ensureExists(CLUSTER_STATE, zkClient);
+ cmdExecutor.ensureExists(ALIASES, zkClient);
log.info("Updating cluster state from ZooKeeper... ");
@@ -286,6 +294,49 @@ public class ZkStateReader {
liveNodeSet.addAll(liveNodes);
ClusterState clusterState = ClusterState.load(zkClient, liveNodeSet);
this.clusterState = clusterState;
+
+ zkClient.exists(ALIASES,
+ new Watcher() {
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events,
+ // and do not remove the watcher
+ if (EventType.None.equals(event.getType())) {
+ return;
+ }
+ try {
+ synchronized (ZkStateReader.this.getUpdateLock()) {
+ log.info("Updating aliases... ");
+
+ // remake watch
+ final Watcher thisWatch = this;
+ Stat stat = new Stat();
+ byte[] data = zkClient.getData(ALIASES, thisWatch, stat ,
+ true);
+
+ Aliases aliases = ClusterState.load(data);
+
+ ZkStateReader.this.aliases = aliases;
+ }
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED
+ || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+ return;
+ }
+ log.error("", e);
+ throw new ZooKeeperException(
+ SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ return;
+ }
+ }
+
+ }, true);
}
}
@@ -493,5 +544,13 @@ public class ZkStateReader {
public SolrZkClient getZkClient() {
return zkClient;
}
+
+ public void updateAliases() throws KeeperException, InterruptedException {
+ byte[] data = zkClient.getData(ALIASES, null, null, true);
+
+ Aliases aliases = ClusterState.load(data);
+
+ ZkStateReader.this.aliases = aliases;
+ }
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java Thu Mar 7 05:58:43 2013
@@ -28,7 +28,7 @@ public interface CollectionParams
public enum CollectionAction {
- CREATE, DELETE, RELOAD, SYNCSHARD;
+ CREATE, DELETE, RELOAD, SYNCSHARD, CREATEALIAS, DELETEALIAS;
public static CollectionAction get( String p )
{
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java Thu Mar 7 05:58:43 2013
@@ -102,7 +102,9 @@ public interface CoreAdminParams
SPLIT,
PREPRECOVERY,
REQUESTRECOVERY,
- REQUESTSYNCSHARD;
+ REQUESTSYNCSHARD,
+ CREATEALIAS,
+ DELETEALIAS;
public static CoreAdminAction get( String p )
{
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java Thu Mar 7 05:58:43 2013
@@ -149,10 +149,4 @@ public class CloudSolrServerTest extends
indexDoc(doc);
}
- SolrInputDocument getDoc(Object... fields) throws Exception {
- SolrInputDocument doc = new SolrInputDocument();
- addFields(doc, fields);
- return doc;
- }
-
}
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Thu Mar 7 05:58:43 2013
@@ -33,11 +33,13 @@ import java.util.concurrent.atomic.Atomi
import org.apache.http.params.CoreConnectionPNames;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
@@ -51,6 +53,7 @@ import org.apache.solr.common.cloud.Slic
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.junit.After;
@@ -612,6 +615,12 @@ public abstract class AbstractFullDistri
super.waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, verbose);
}
+ protected void waitForRecoveriesToFinish(String collection, boolean verbose)
+ throws Exception {
+ ZkStateReader zkStateReader = cloudClient.getZkStateReader();
+ super.waitForRecoveriesToFinish(collection, zkStateReader, verbose);
+ }
+
protected void waitForRecoveriesToFinish(boolean verbose, int timeoutSeconds)
throws Exception {
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
@@ -1318,6 +1327,9 @@ public abstract class AbstractFullDistri
if (VERBOSE || printLayoutOnTearDown) {
super.printLayout();
}
+ if (commondCloudSolrServer != null) {
+ commondCloudSolrServer.shutdown();
+ }
if (controlClient != null) {
((HttpSolrServer) controlClient).shutdown();
}
@@ -1355,6 +1367,40 @@ public abstract class AbstractFullDistri
jettys.clear();
}
+ protected void createCollection(String collectionName, int numShards, int numReplicas, int maxShardsPerNode) throws SolrServerException, IOException {
+ createCollection(null, collectionName, numShards, numReplicas, maxShardsPerNode, null, null);
+ }
+
+ protected void createCollection(Map<String,List<Integer>> collectionInfos,
+ String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionAction.CREATE.toString());
+
+ params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
+ params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
+ params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
+ if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
+
+ int clientIndex = random().nextInt(2);
+ List<Integer> list = new ArrayList<Integer>();
+ list.add(numShards);
+ list.add(numReplicas);
+ if (collectionInfos != null) {
+ collectionInfos.put(collectionName, list);
+ }
+ params.set("name", collectionName);
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ if (client == null) {
+ final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(clientIndex));
+
+ createNewSolrServer("", baseUrl).request(request);
+ } else {
+ client.request(request);
+ }
+ }
+
@Override
protected SolrServer createNewSolrServer(int port) {
try {
@@ -1371,5 +1417,103 @@ public abstract class AbstractFullDistri
throw new RuntimeException(ex);
}
}
+
+ protected SolrServer createNewSolrServer(String collection, String baseUrl) {
+ try {
+ // setup the server...
+ HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
+ s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
+ s.setDefaultMaxConnectionsPerHost(100);
+ s.setMaxTotalConnections(100);
+ return s;
+ }
+ catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ protected String getBaseUrl(HttpSolrServer client) {
+ return client .getBaseURL().substring(
+ 0, client.getBaseURL().length()
+ - DEFAULT_COLLECTION.length() - 1);
+ }
+
+ protected SolrInputDocument getDoc(Object... fields) throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ addFields(doc, fields);
+ return doc;
+ }
+
+ private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
+ ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+
+ int expectedSlices = numShardsNumReplicaList.get(0);
+ // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
+ int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
+ int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
+
+ Map<String,DocCollection> collections = clusterState
+ .getCollectionStates();
+ if (collections.containsKey(collectionName)) {
+ Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
+ // did we find expectedSlices slices/shards?
+ if (slices.size() != expectedSlices) {
+ return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
+ }
+ int totalShards = 0;
+ for (String sliceName : slices.keySet()) {
+ for (Replica replica : slices.get(sliceName).getReplicas()) {
+ if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
+ return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
+ }
+ }
+ totalShards += slices.get(sliceName).getReplicas().size();
+ }
+ if (totalShards != expectedTotalShards) {
+ return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
+ }
+ return null;
+ } else {
+ return "Could not find new collection " + collectionName;
+ }
+ }
+
+ protected void checkForCollection(String collectionName,
+ List<Integer> numShardsNumReplicaList,
+ List<String> nodesAllowedToRunShards) throws Exception {
+ // check for an expectedSlices new collection - we poll the state
+ long timeoutAt = System.currentTimeMillis() + 120000;
+ boolean success = false;
+ String checkResult = "Didnt get to perform a single check";
+ while (System.currentTimeMillis() < timeoutAt) {
+ checkResult = checkCollectionExpectations(collectionName,
+ numShardsNumReplicaList, nodesAllowedToRunShards);
+ if (checkResult == null) {
+ success = true;
+ break;
+ }
+ Thread.sleep(500);
+ }
+ if (!success) {
+ super.printLayout();
+ fail(checkResult);
+ }
+ }
+
+ volatile CloudSolrServer commondCloudSolrServer;
+ protected CloudSolrServer getCommonCloudSolrServer() {
+ if (commondCloudSolrServer == null) {
+ synchronized(this) {
+ try {
+ commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
+ commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
+ commondCloudSolrServer.connect();
+ } catch (MalformedURLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ return commondCloudSolrServer;
+ }
}