You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2013/03/07 06:58:43 UTC

svn commit: r1453691 - in /lucene/dev/branches/branch_4x: ./ solr/ solr/core/ solr/core/src/java/org/apache/solr/cloud/ solr/core/src/java/org/apache/solr/handler/admin/ solr/core/src/java/org/apache/solr/servlet/ solr/core/src/test/org/apache/solr/clo...

Author: markrmiller
Date: Thu Mar  7 05:58:43 2013
New Revision: 1453691

URL: http://svn.apache.org/r1453691
Log:
SOLR-4497: Collection Aliasing.

Added:
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
      - copied unchanged from r1453687, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
      - copied unchanged from r1453687, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
    lucene/dev/branches/branch_4x/solr/solrj/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
    lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Thu Mar  7 05:58:43 2013
@@ -200,6 +200,8 @@ Bug Fixes
   dynamic field references that aren't string-equal to the name of
   the referenced dynamic field. (Steve Rowe)
 
+* SOLR-4497: Collection Aliasing. (Mark Miller)
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Thu Mar  7 05:58:43 2013
@@ -19,6 +19,7 @@ package org.apache.solr.cloud;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -27,6 +28,7 @@ import org.apache.solr.client.solrj.Solr
 import org.apache.solr.cloud.DistributedQueue.QueueEvent;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Aliases;
 import org.apache.solr.common.cloud.ClosableThread;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.DocCollection;
@@ -63,10 +65,12 @@ public class OverseerCollectionProcessor
 
   public static final String RELOADCOLLECTION = "reloadcollection";
   
+  public static final String CREATEALIAS = "createalias";
+  
+  public static final String DELETEALIAS = "deletealias";
+  
   // TODO: use from Overseer?
   private static final String QUEUE_OPERATION = "operation";
-
-
   
   private static Logger log = LoggerFactory
       .getLogger(OverseerCollectionProcessor.class);
@@ -162,6 +166,10 @@ public class OverseerCollectionProcessor
         ModifiableSolrParams params = new ModifiableSolrParams();
         params.set(CoreAdminParams.ACTION, CoreAdminAction.RELOAD.toString());
         collectionCmd(zkStateReader.getClusterState(), message, params);
+      } else if (CREATEALIAS.equals(operation)) {
+        createAlias(zkStateReader.getAliases(), message);
+      } else if (DELETEALIAS.equals(operation)) {
+        deleteAlias(zkStateReader.getAliases(), message);
       } else {
         throw new SolrException(ErrorCode.BAD_REQUEST, "Unknow the operation:"
             + operation);
@@ -182,15 +190,116 @@ public class OverseerCollectionProcessor
           }
         }
       } while (srsp != null);
-    } catch (SolrException ex) {
+    } catch (Exception ex) {
       SolrException.log(log, "Collection " + operation + " of " + operation
-          + " failed");
+          + " failed", ex);
       results.add("Operation " + operation + " caused exception:", ex);
     } finally {
       return new OverseerSolrResponse(results);
     }
   }
 
+  private void createAlias(Aliases aliases, ZkNodeProps message) {
+    String aliasName = message.getStr("name");
+    String collections = message.getStr("collections");
+    
+    Map<String,Map<String,String>> newAliasesMap = new HashMap<String,Map<String,String>>();
+    Map<String,String> newCollectionAliasesMap = new HashMap<String,String>();
+    Map<String,String> prevColAliases = aliases.getCollectionAliasMap();
+    if (prevColAliases != null) {
+      newCollectionAliasesMap.putAll(prevColAliases);
+    }
+    newCollectionAliasesMap.put(aliasName, collections);
+    newAliasesMap.put("collection", newCollectionAliasesMap);
+    Aliases newAliases = new Aliases(newAliasesMap);
+    byte[] jsonBytes = null;
+    if (newAliases.collectionAliasSize() > 0) { // only sub map right now
+      jsonBytes  = ZkStateReader.toJSON(newAliases.getAliasMap());
+    }
+    try {
+      zkStateReader.getZkClient().setData(ZkStateReader.ALIASES,
+          jsonBytes, true);
+      
+      checkForAlias(aliasName, collections);
+      // some fudge for other nodes
+      Thread.sleep(100);
+    } catch (KeeperException e) {
+      log.error("", e);
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    } catch (InterruptedException e) {
+      log.warn("", e);
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    }
+
+  }
+  
+  private void checkForAlias(String name, String value) {
+
+    long now = System.currentTimeMillis();
+    long timeout = now + 30000;
+    boolean success = false;
+    Aliases aliases = null;
+    while (System.currentTimeMillis() < timeout) {
+      aliases = zkStateReader.getAliases();
+      String collections = aliases.getCollectionAlias(name);
+      if (collections != null && collections.equals(value)) {
+        success = true;
+        break;
+      }
+    }
+    if (!success) {
+      log.warn("Timeout waiting to be notified of Alias change...");
+    }
+  }
+  
+  private void checkForAliasAbsence(String name) {
+
+    long now = System.currentTimeMillis();
+    long timeout = now + 30000;
+    boolean success = false;
+    Aliases aliases = null;
+    while (System.currentTimeMillis() < timeout) {
+      aliases = zkStateReader.getAliases();
+      String collections = aliases.getCollectionAlias(name);
+      if (collections == null) {
+        success = true;
+        break;
+      }
+    }
+    if (!success) {
+      log.warn("Timeout waiting to be notified of Alias change...");
+    }
+  }
+
+  private void deleteAlias(Aliases aliases, ZkNodeProps message) {
+    String aliasName = message.getStr("name");
+
+    Map<String,Map<String,String>> newAliasesMap = new HashMap<String,Map<String,String>>();
+    Map<String,String> newCollectionAliasesMap = new HashMap<String,String>();
+    newCollectionAliasesMap.putAll(aliases.getCollectionAliasMap());
+    newCollectionAliasesMap.remove(aliasName);
+    newAliasesMap.put("collection", newCollectionAliasesMap);
+    Aliases newAliases = new Aliases(newAliasesMap);
+    byte[] jsonBytes = null;
+    if (newAliases.collectionAliasSize() > 0) { // only sub map right now
+      jsonBytes  = ZkStateReader.toJSON(newAliases.getAliasMap());
+    }
+    try {
+      zkStateReader.getZkClient().setData(ZkStateReader.ALIASES,
+          jsonBytes, true);
+      checkForAliasAbsence(aliasName);
+      // some fudge for other nodes
+      Thread.sleep(100);
+    } catch (KeeperException e) {
+      log.error("", e);
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    } catch (InterruptedException e) {
+      log.warn("", e);
+      throw new SolrException(ErrorCode.SERVER_ERROR, e);
+    }
+    
+  }
+  
   private boolean createCollection(ClusterState clusterState, ZkNodeProps message) {
     String collectionName = message.getStr("name");
     if (clusterState.getCollections().contains(collectionName)) {

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Thu Mar  7 05:58:43 2013
@@ -120,7 +120,14 @@ public class CollectionsHandler extends 
           this.handleSyncShardAction(req, rsp);
           break;
         }
-        
+        case CREATEALIAS: {
+          this.handleCreateAliasAction(req, rsp);
+          break;
+        }
+        case DELETEALIAS: {
+          this.handleDeleteAliasAction(req, rsp);
+          break;
+        }
         default: {
           throw new RuntimeException("Unknown action: " + action);
         }
@@ -187,7 +194,30 @@ public class CollectionsHandler extends 
     reqSyncShard.setCoreName(nodeProps.getCoreName());
     server.request(reqSyncShard);
   }
-
+  
+  private void handleCreateAliasAction(SolrQueryRequest req,
+      SolrQueryResponse rsp) throws Exception {
+    log.info("Create alias action : " + req.getParamString());
+    String name = req.getParams().required().get("name");
+    String collections = req.getParams().required().get("collections");
+    
+    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
+        OverseerCollectionProcessor.CREATEALIAS, "name", name, "collections",
+        collections);
+    
+    handleResponse(OverseerCollectionProcessor.CREATEALIAS, m, rsp);
+  }
+  
+  private void handleDeleteAliasAction(SolrQueryRequest req,
+      SolrQueryResponse rsp) throws Exception {
+    log.info("Delete alias action : " + req.getParamString());
+    String name = req.getParams().required().get("name");
+    
+    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
+        OverseerCollectionProcessor.DELETEALIAS, "name", name);
+    
+    handleResponse(OverseerCollectionProcessor.CREATEALIAS, m, rsp);
+  }
 
   private void handleDeleteAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
     log.info("Deleting Collection : " + req.getParamString());
@@ -200,7 +230,6 @@ public class CollectionsHandler extends 
     handleResponse(OverseerCollectionProcessor.DELETECOLLECTION, m, rsp);
   }
 
-
   // very simple currently, you can pass a template collection, and the new collection is created on
   // every node the template collection is on
   // there is a lot more to add - you should also be able to create with an explicit server list

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Thu Mar  7 05:58:43 2013
@@ -17,8 +17,6 @@
 
 package org.apache.solr.servlet;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -31,6 +29,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Enumeration;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -42,7 +41,6 @@ import javax.servlet.Filter;
 import javax.servlet.FilterChain;
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
-import javax.servlet.ServletOutputStream;
 import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
@@ -50,6 +48,7 @@ import javax.servlet.http.HttpServletRes
 
 import org.apache.commons.io.IOUtils;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Aliases;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
@@ -58,10 +57,12 @@ import org.apache.solr.common.cloud.ZkNo
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrCore;
@@ -134,9 +135,13 @@ public class SolrDispatchFilter implemen
       cores = null;
     }    
   }
-
+  
   @Override
   public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
+    doFilter(request, response, chain, false);
+  }
+  
+  public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain, boolean retry) throws IOException, ServletException {
     if( abortErrorMessage != null ) {
       ((HttpServletResponse)response).sendError( 500, abortErrorMessage );
       return;
@@ -149,12 +154,14 @@ public class SolrDispatchFilter implemen
     CoreContainer cores = this.cores;
     SolrCore core = null;
     SolrQueryRequest solrReq = null;
+    Aliases aliases = null;
     
     if( request instanceof HttpServletRequest) {
       HttpServletRequest req = (HttpServletRequest)request;
       HttpServletResponse resp = (HttpServletResponse)response;
       SolrRequestHandler handler = null;
       String corename = "";
+      String origCorename = null;
       try {
         // put the core container in request attribute
         req.setAttribute("org.apache.solr.CoreContainer", cores);
@@ -185,6 +192,8 @@ public class SolrDispatchFilter implemen
           handleAdminRequest(req, response, handler, solrReq);
           return;
         }
+        boolean usingAliases = false;
+        List<String> collectionsList = null;
         // Check for the core admin collections url
         if( path.equals( "/admin/collections" ) ) {
           handler = cores.getCollectionsHandler();
@@ -198,7 +207,24 @@ public class SolrDispatchFilter implemen
           if( idx > 1 ) {
             // try to get the corename as a request parameter first
             corename = path.substring( 1, idx );
+            
+            // look at aliases
+            if (cores.isZooKeeperAware()) {
+              origCorename = corename;
+              ZkStateReader reader = cores.getZkController().getZkStateReader();
+              aliases = reader.getAliases();
+              if (aliases != null && aliases.collectionAliasSize() > 0) {
+                usingAliases = true;
+                String alias = aliases.getCollectionAlias(corename);
+                if (alias != null) {
+                  collectionsList = StrUtils.splitSmart(alias, ",", true);
+                  corename = collectionsList.get(0);
+                }
+              }
+            }
+            
             core = cores.getCore(corename);
+
             if (core != null) {
               path = path.substring( idx );
             }
@@ -221,11 +247,19 @@ public class SolrDispatchFilter implemen
           
           // if we couldn't find it locally, look on other nodes
           if (core == null && idx > 0) {
-            String coreUrl = getRemotCoreUrl(cores, corename);
+            String coreUrl = getRemotCoreUrl(cores, corename, origCorename);
             if (coreUrl != null) {
               path = path.substring( idx );
               remoteQuery(coreUrl + path, req, solrReq, resp);
               return;
+            } else {
+              if (!retry) {
+                // we couldn't find a core to work with, try reloading aliases
+                ZkStateReader reader = cores.getZkController()
+                    .getZkStateReader();
+                reader.updateAliases();
+                doFilter(request, response, chain, true);
+              }
             }
           }
           
@@ -289,6 +323,10 @@ public class SolrDispatchFilter implemen
               solrReq = parser.parse( core, path, req );
             }
 
+            if (usingAliases) {
+              processAliases(solrReq, aliases, collectionsList);
+            }
+            
             final Method reqMethod = Method.getMethod(req.getMethod());
             HttpCacheHeaderUtil.setCacheControlHeader(config, resp, reqMethod);
             // unless we have been explicitly told not to, do cache validation
@@ -341,9 +379,44 @@ public class SolrDispatchFilter implemen
     chain.doFilter(request, response);
   }
   
+  private void processAliases(SolrQueryRequest solrReq, Aliases aliases,
+      List<String> collectionsList) {
+    String collection = solrReq.getParams().get("collection");
+    if (collection != null) {
+      collectionsList = StrUtils.splitSmart(collection, ",", true);
+    }
+    if (collectionsList != null) {
+      Set<String> newCollectionsList = new HashSet<String>(
+          collectionsList.size());
+      for (String col : collectionsList) {
+        String al = aliases.getCollectionAlias(col);
+        if (al != null) {
+          List<String> aliasList = StrUtils.splitSmart(al, ",", true);
+          newCollectionsList.addAll(aliasList);
+        } else {
+          newCollectionsList.add(col);
+        }
+      }
+      if (newCollectionsList.size() > 0) {
+        StringBuilder collectionString = new StringBuilder();
+        Iterator<String> it = newCollectionsList.iterator();
+        int sz = newCollectionsList.size();
+        for (int i = 0; i < sz; i++) {
+          collectionString.append(it.next());
+          if (i < newCollectionsList.size() - 1) {
+            collectionString.append(",");
+          }
+        }
+        ModifiableSolrParams params = new ModifiableSolrParams(
+            solrReq.getParams());
+        params.set("collection", collectionString.toString());
+        solrReq.setParams(params);
+      }
+    }
+  }
+  
   private void remoteQuery(String coreUrl, HttpServletRequest req,
       SolrQueryRequest solrReq, HttpServletResponse resp) throws IOException {
-    
     try {
       String urlstr = coreUrl;
       
@@ -409,7 +482,7 @@ public class SolrDispatchFilter implemen
     
   }
   
-  private String getRemotCoreUrl(CoreContainer cores, String collectionName) {
+  private String getRemotCoreUrl(CoreContainer cores, String collectionName, String origCorename) {
     ClusterState clusterState = cores.getZkController().getClusterState();
     Collection<Slice> slices = clusterState.getSlices(collectionName);
     boolean byCoreName = false;
@@ -444,9 +517,14 @@ public class SolrDispatchFilter implemen
             // don't count a local core
             continue;
           }
-          String coreUrl = coreNodeProps.getCoreUrl();
-          if (coreUrl.endsWith("/")) {
-            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
+          String coreUrl;
+          if (origCorename != null) {
+            coreUrl = coreNodeProps.getBaseUrl() + "/" + origCorename;
+          } else {
+            coreUrl = coreNodeProps.getCoreUrl();
+            if (coreUrl.endsWith("/")) {
+              coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
+            }
           }
 
           return coreUrl;

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Thu Mar  7 05:58:43 2013
@@ -330,12 +330,5 @@ public class ChaosMonkeyNothingIsSafeTes
     SolrInputDocument doc = getDoc(fields);
     indexDoc(doc);
   }
-
-  SolrInputDocument getDoc(Object... fields) throws Exception {
-    SolrInputDocument doc = new SolrInputDocument();
-    addFields(doc, fields);
-    addFields(doc, "rnd_b", true);
-    return doc;
-  }
   
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Thu Mar  7 05:58:43 2013
@@ -44,8 +44,6 @@ import javax.management.ObjectName;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.lucene.util._TestUtil;
 import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrServer;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -149,10 +147,7 @@ public class CollectionsAPIDistributedZk
 
   private void testNodesUsedByCreate() throws Exception {
     // we can use this client because we just want base url
-    final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
-        0,
-        ((HttpSolrServer) clients.get(0)).getBaseURL().length()
-            - DEFAULT_COLLECTION.length() - 1);
+    final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
     
     ModifiableSolrParams params = new ModifiableSolrParams();
     params.set("action", CollectionAction.CREATE.toString());
@@ -297,10 +292,7 @@ public class CollectionsAPIDistributedZk
     request.setPath("/admin/collections");
     
     // we can use this client because we just want base url
-    final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
-        0,
-        ((HttpSolrServer) clients.get(0)).getBaseURL().length()
-            - DEFAULT_COLLECTION.length() - 1);
+    final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
     
     createNewSolrServer("", baseUrl).request(request);
 
@@ -409,38 +401,6 @@ public class CollectionsAPIDistributedZk
     checkNoTwoShardsUseTheSameIndexDir();
   }
 
-
-  protected void createCollection(Map<String,List<Integer>> collectionInfos,
-      String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
-    ModifiableSolrParams params = new ModifiableSolrParams();
-    params.set("action", CollectionAction.CREATE.toString());
-
-    params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
-    params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
-    params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
-    if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
-
-    int clientIndex = random().nextInt(2);
-    List<Integer> list = new ArrayList<Integer>();
-    list.add(numShards);
-    list.add(numReplicas);
-    collectionInfos.put(collectionName, list);
-    params.set("name", collectionName);
-    SolrRequest request = new QueryRequest(params);
-    request.setPath("/admin/collections");
-  
-    if (client == null) {
-      final String baseUrl = ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().substring(
-          0,
-          ((HttpSolrServer) clients.get(clientIndex)).getBaseURL().length()
-              - DEFAULT_COLLECTION.length() - 1);
-      
-      createNewSolrServer("", baseUrl).request(request);
-    } else {
-      client.request(request);
-    }
-  }
-
   private boolean waitForReloads(String collectionName, Map<String,Long> urlToTimeBefore) throws SolrServerException, IOException {
     
     
@@ -549,60 +509,6 @@ public class CollectionsAPIDistributedZk
     fail("Could not find the new collection - " + exp.code() + " : " + collectionClient.getBaseURL());
   }
 
-  private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
-    ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-    
-    int expectedSlices = numShardsNumReplicaList.get(0);
-    // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
-    int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
-    int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
-    
-      Map<String,DocCollection> collections = clusterState
-          .getCollectionStates();
-      if (collections.containsKey(collectionName)) {
-        Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
-        // did we find expectedSlices slices/shards?
-      if (slices.size() != expectedSlices) {
-        return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
-      }
-      int totalShards = 0;
-      for (String sliceName : slices.keySet()) {
-        for (Replica replica : slices.get(sliceName).getReplicas()) {
-          if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
-            return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
-          }
-        }
-        totalShards += slices.get(sliceName).getReplicas().size();
-      }
-      if (totalShards != expectedTotalShards) {
-        return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards; 
-        }
-      return null;
-    } else {
-      return "Could not find new collection " + collectionName;
-    }
-  }
-  
-  private void checkForCollection(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards)
-      throws Exception {
-    // check for an expectedSlices new collection - we poll the state
-    long timeoutAt = System.currentTimeMillis() + 120000;
-    boolean success = false;
-    String checkResult = "Didnt get to perform a single check";
-    while (System.currentTimeMillis() < timeoutAt) {
-      checkResult = checkCollectionExpectations(collectionName, numShardsNumReplicaList, nodesAllowedToRunShards);
-      if (checkResult == null) {
-        success = true;
-        break;
-      }
-      Thread.sleep(500);
-    }
-    if (!success) {
-      super.printLayout();
-      fail(checkResult);
-      }
-    }
-
   private void checkCollectionIsNotCreated(String collectionName)
     throws Exception {
     // TODO: REMOVE THIS SLEEP WHEN WE HAVE COLLECTION API RESPONSES
@@ -678,42 +584,6 @@ public class CollectionsAPIDistributedZk
     }
 
   }
-  
-  protected SolrInputDocument getDoc(Object... fields) throws Exception {
-    SolrInputDocument doc = new SolrInputDocument();
-    addFields(doc, fields);
-    return doc;
-  }
-  
-  protected SolrServer createNewSolrServer(String collection, String baseUrl) {
-    try {
-      // setup the server...
-      HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
-      s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
-      s.setDefaultMaxConnectionsPerHost(100);
-      s.setMaxTotalConnections(100);
-      return s;
-    }
-    catch (Exception ex) {
-      throw new RuntimeException(ex);
-    }
-  }
-
-  volatile CloudSolrServer commondCloudSolrServer;
-  private CloudSolrServer getCommonCloudSolrServer() {
-    if (commondCloudSolrServer == null) {
-      synchronized(this) {
-        try {
-          commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
-          commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
-          commondCloudSolrServer.connect();
-        } catch (MalformedURLException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-    return commondCloudSolrServer;
-  }
 
   @Override
   protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
@@ -731,9 +601,6 @@ public class CollectionsAPIDistributedZk
   @Override
   public void tearDown() throws Exception {
     super.tearDown();
-    if (commondCloudSolrServer != null) {
-      commondCloudSolrServer.shutdown();
-    }
     System.clearProperty("numShards");
     System.clearProperty("zkHost");
     System.clearProperty("solr.xml.persist");

Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Thu Mar  7 05:58:43 2013
@@ -19,10 +19,13 @@ package org.apache.solr.client.solrj.imp
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -36,6 +39,8 @@ import org.apache.solr.client.solrj.Solr
 import org.apache.solr.client.solrj.request.IsUpdateRequest;
 import org.apache.solr.client.solrj.util.ClientUtils;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Aliases;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -183,13 +188,13 @@ public class CloudSolrServer extends Sol
       reqParams = new ModifiableSolrParams();
     }
     List<String> theUrlList = new ArrayList<String>();
-    if (request.getPath().equals("/admin/collections")) {
+    if (request.getPath().equals("/admin/collections") || request.getPath().equals("/admin/cores")) {
       Set<String> liveNodes = clusterState.getLiveNodes();
       for (String liveNode : liveNodes) {
         int splitPointBetweenHostPortAndContext = liveNode.indexOf("_");
         theUrlList.add("http://"
             + liveNode.substring(0, splitPointBetweenHostPortAndContext) + "/"
-            + liveNode.substring(splitPointBetweenHostPortAndContext + 1));
+            + URLDecoder.decode(liveNode, "UTF-8").substring(splitPointBetweenHostPortAndContext + 1));
       }
     } else {
       String collection = reqParams.get("collection", defaultCollection);
@@ -199,9 +204,21 @@ public class CloudSolrServer extends Sol
             "No collection param specified on request and no default collection has been set.");
       }
       
-      // Extract each comma separated collection name and store in a List.
-      List<String> collectionList = StrUtils.splitSmart(collection, ",", true);
+      Set<String> collectionsList = getCollectionList(clusterState, collection);
+      if (collectionsList.size() == 0) {
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
+      }
+      collection = collectionsList.iterator().next();
       
+      StringBuilder collectionString = new StringBuilder();
+      Iterator<String> it = collectionsList.iterator();
+      for (int i = 0; i < collectionsList.size(); i++) {
+        String col = it.next(); 
+        collectionString.append(col);
+        if (i < collectionsList.size() - 1) {
+          collectionString.append(",");
+        }
+      }
       // TODO: not a big deal because of the caching, but we could avoid looking
       // at every shard
       // when getting leaders if we tweaked some things
@@ -210,9 +227,12 @@ public class CloudSolrServer extends Sol
       // specified,
       // add it to the Map of slices.
       Map<String,Slice> slices = new HashMap<String,Slice>();
-      for (String collectionName : collectionList) {
-        ClientUtils.addSlices(slices, collectionName,
-            clusterState.getSlices(collectionName), true);
+      for (String collectionName : collectionsList) {
+        Collection<Slice> colSlices = clusterState.getSlices(collectionName);
+        if (colSlices == null) {
+          throw new SolrServerException("Could not find collection:" + collectionName);
+        }
+        ClientUtils.addSlices(slices, collectionName, colSlices, true);
       }
       Set<String> liveNodes = clusterState.getLiveNodes();
       
@@ -287,6 +307,30 @@ public class CloudSolrServer extends Sol
     return rsp.getResponse();
   }
 
+  private Set<String> getCollectionList(ClusterState clusterState,
+      String collection) {
+    // Extract each comma separated collection name and store in a List.
+    List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
+    Set<String> collectionsList = new HashSet<String>();
+    // validate collections
+    for (String collectionName : rawCollectionsList) {
+      if (!clusterState.getCollections().contains(collectionName)) {
+        Aliases aliases = zkStateReader.getAliases();
+        String alias = aliases.getCollectionAlias(collectionName);
+        if (alias != null) {
+          List<String> aliasList = StrUtils.splitSmart(alias, ",", true); 
+          collectionsList.addAll(aliasList);
+          continue;
+        }
+        
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
+      }
+      
+      collectionsList.add(collectionName);
+    }
+    return collectionsList;
+  }
+
   @Override
   public void shutdown() {
     if (zkStateReader != null) {

Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java Thu Mar  7 05:58:43 2013
@@ -17,13 +17,11 @@ package org.apache.solr.common.cloud;
  * limitations under the License.
  */
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -31,7 +29,6 @@ import java.util.Set;
 import org.apache.noggit.JSONWriter;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.DocRouter.Range;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
@@ -229,6 +226,15 @@ public class ClusterState implements JSO
     // System.out.println("######## ClusterState.load result:" + collections);
     return new ClusterState(version, liveNodes, collections);
   }
+  
+  public static Aliases load(byte[] bytes) {
+    if (bytes == null || bytes.length == 0) {
+      return new Aliases();
+    }
+    Map<String,Map<String,String>> aliasMap = (Map<String,Map<String,String>>) ZkStateReader.fromJSON(bytes);
+
+    return new Aliases(aliasMap);
+  }
 
   private static DocCollection collectionFromObjects(String name, Map<String,Object> objs) {
     Map<String,Object> props;

Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Thu Mar  7 05:58:43 2013
@@ -62,6 +62,7 @@ public class ZkStateReader {
   
   public static final String COLLECTIONS_ZKNODE = "/collections";
   public static final String LIVE_NODES_ZKNODE = "/live_nodes";
+  public static final String ALIASES = "/aliases.json";
   public static final String CLUSTER_STATE = "/clusterstate.json";
   
   public static final String RECOVERING = "recovering";
@@ -129,7 +130,9 @@ public class ZkStateReader {
   private boolean closeClient = false;
 
   private ZkCmdExecutor cmdExecutor;
-  
+
+  private Aliases aliases = new Aliases();
+
   public ZkStateReader(SolrZkClient zkClient) {
     this.zkClient = zkClient;
     initZkCmdExecutor(zkClient.getZkClientTimeout());
@@ -177,12 +180,17 @@ public class ZkStateReader {
     updateClusterState(true, true);
   }
   
+  public Aliases getAliases() {
+    return aliases;
+  }
+  
   public synchronized void createClusterStateWatchersAndUpdate() throws KeeperException,
       InterruptedException {
     // We need to fetch the current cluster state and the set of live nodes
     
     synchronized (getUpdateLock()) {
       cmdExecutor.ensureExists(CLUSTER_STATE, zkClient);
+      cmdExecutor.ensureExists(ALIASES, zkClient);
       
       log.info("Updating cluster state from ZooKeeper... ");
       
@@ -286,6 +294,49 @@ public class ZkStateReader {
       liveNodeSet.addAll(liveNodes);
       ClusterState clusterState = ClusterState.load(zkClient, liveNodeSet);
       this.clusterState = clusterState;
+      
+      zkClient.exists(ALIASES,
+          new Watcher() {
+            
+            @Override
+            public void process(WatchedEvent event) {
+              // session events are not change events,
+              // and do not remove the watcher
+              if (EventType.None.equals(event.getType())) {
+                return;
+              }
+              try {
+                synchronized (ZkStateReader.this.getUpdateLock()) {
+                  log.info("Updating aliases... ");
+
+                  // remake watch
+                  final Watcher thisWatch = this;
+                  Stat stat = new Stat();
+                  byte[] data = zkClient.getData(ALIASES, thisWatch, stat ,
+                      true);
+
+                  Aliases aliases = ClusterState.load(data);
+
+                  ZkStateReader.this.aliases = aliases;
+                }
+              } catch (KeeperException e) {
+                if (e.code() == KeeperException.Code.SESSIONEXPIRED
+                    || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+                  log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+                  return;
+                }
+                log.error("", e);
+                throw new ZooKeeperException(
+                    SolrException.ErrorCode.SERVER_ERROR, "", e);
+              } catch (InterruptedException e) {
+                // Restore the interrupted status
+                Thread.currentThread().interrupt();
+                log.warn("", e);
+                return;
+              }
+            }
+            
+          }, true);
     }
   }
   
@@ -493,5 +544,13 @@ public class ZkStateReader {
   public SolrZkClient getZkClient() {
     return zkClient;
   }
+
+  public void updateAliases() throws KeeperException, InterruptedException {
+    byte[] data = zkClient.getData(ALIASES, null, null, true);
+
+    Aliases aliases = ClusterState.load(data);
+
+    ZkStateReader.this.aliases = aliases;
+  }
   
 }

Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java Thu Mar  7 05:58:43 2013
@@ -28,7 +28,7 @@ public interface CollectionParams 
 
 
   public enum CollectionAction {
-    CREATE, DELETE, RELOAD, SYNCSHARD;
+    CREATE, DELETE, RELOAD, SYNCSHARD, CREATEALIAS, DELETEALIAS;
     
     public static CollectionAction get( String p )
     {

Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java Thu Mar  7 05:58:43 2013
@@ -102,7 +102,9 @@ public interface CoreAdminParams 
     SPLIT,
     PREPRECOVERY,
     REQUESTRECOVERY, 
-    REQUESTSYNCSHARD;
+    REQUESTSYNCSHARD,
+    CREATEALIAS,
+    DELETEALIAS;
     
     public static CoreAdminAction get( String p )
     {

Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java Thu Mar  7 05:58:43 2013
@@ -149,10 +149,4 @@ public class CloudSolrServerTest extends
     indexDoc(doc);
   }
 
-  SolrInputDocument getDoc(Object... fields) throws Exception {
-    SolrInputDocument doc = new SolrInputDocument();
-    addFields(doc, fields);
-    return doc;
-  }
-
 }

Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1453691&r1=1453690&r2=1453691&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Thu Mar  7 05:58:43 2013
@@ -33,11 +33,13 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.http.params.CoreConnectionPNames;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.impl.CloudSolrServer;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
@@ -51,6 +53,7 @@ import org.apache.solr.common.cloud.Slic
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.junit.After;
@@ -612,6 +615,12 @@ public abstract class AbstractFullDistri
     super.waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, verbose);
   }
   
+  protected void waitForRecoveriesToFinish(String collection, boolean verbose)
+      throws Exception {
+    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
+    super.waitForRecoveriesToFinish(collection, zkStateReader, verbose);
+  }
+  
   protected void waitForRecoveriesToFinish(boolean verbose, int timeoutSeconds)
       throws Exception {
     ZkStateReader zkStateReader = cloudClient.getZkStateReader();
@@ -1318,6 +1327,9 @@ public abstract class AbstractFullDistri
     if (VERBOSE || printLayoutOnTearDown) {
       super.printLayout();
     }
+    if (commondCloudSolrServer != null) {
+      commondCloudSolrServer.shutdown();
+    }
     if (controlClient != null) {
       ((HttpSolrServer) controlClient).shutdown();
     }
@@ -1355,6 +1367,40 @@ public abstract class AbstractFullDistri
     jettys.clear();
   }
   
+  protected void createCollection(String collectionName, int numShards, int numReplicas, int maxShardsPerNode) throws SolrServerException, IOException {
+    createCollection(null, collectionName, numShards, numReplicas, maxShardsPerNode, null, null);
+  }
+  
+  protected void createCollection(Map<String,List<Integer>> collectionInfos,
+      String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr) throws SolrServerException, IOException {
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set("action", CollectionAction.CREATE.toString());
+
+    params.set(OverseerCollectionProcessor.NUM_SLICES, numShards);
+    params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas);
+    params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
+    if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
+
+    int clientIndex = random().nextInt(2);
+    List<Integer> list = new ArrayList<Integer>();
+    list.add(numShards);
+    list.add(numReplicas);
+    if (collectionInfos != null) {
+      collectionInfos.put(collectionName, list);
+    }
+    params.set("name", collectionName);
+    SolrRequest request = new QueryRequest(params);
+    request.setPath("/admin/collections");
+  
+    if (client == null) {
+      final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(clientIndex));
+      
+      createNewSolrServer("", baseUrl).request(request);
+    } else {
+      client.request(request);
+    }
+  }
+  
   @Override
   protected SolrServer createNewSolrServer(int port) {
     try {
@@ -1371,5 +1417,103 @@ public abstract class AbstractFullDistri
       throw new RuntimeException(ex);
     }
   }
+  
+  protected SolrServer createNewSolrServer(String collection, String baseUrl) {
+    try {
+      // setup the server...
+      HttpSolrServer s = new HttpSolrServer(baseUrl + "/" + collection);
+      s.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
+      s.setDefaultMaxConnectionsPerHost(100);
+      s.setMaxTotalConnections(100);
+      return s;
+    }
+    catch (Exception ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+  
+  protected String getBaseUrl(HttpSolrServer client) {
+    return client .getBaseURL().substring(
+        0, client.getBaseURL().length()
+            - DEFAULT_COLLECTION.length() - 1);
+  }
+  
+  protected SolrInputDocument getDoc(Object... fields) throws Exception {
+    SolrInputDocument doc = new SolrInputDocument();
+    addFields(doc, fields);
+    return doc;
+  }
+
+  private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
+    ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
+    
+    int expectedSlices = numShardsNumReplicaList.get(0);
+    // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
+    int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
+    int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
+    
+      Map<String,DocCollection> collections = clusterState
+          .getCollectionStates();
+      if (collections.containsKey(collectionName)) {
+        Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
+        // did we find expectedSlices slices/shards?
+      if (slices.size() != expectedSlices) {
+        return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
+      }
+      int totalShards = 0;
+      for (String sliceName : slices.keySet()) {
+        for (Replica replica : slices.get(sliceName).getReplicas()) {
+          if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
+            return "Shard " + replica.getName() + " created on node " + replica.getStr(ZkStateReader.NODE_NAME_PROP) + " not allowed to run shards for the created collection " + collectionName;
+          }
+        }
+        totalShards += slices.get(sliceName).getReplicas().size();
+      }
+      if (totalShards != expectedTotalShards) {
+        return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards; 
+        }
+      return null;
+    } else {
+      return "Could not find new collection " + collectionName;
+    }
+  }
+  
+  protected void checkForCollection(String collectionName,
+      List<Integer> numShardsNumReplicaList,
+      List<String> nodesAllowedToRunShards) throws Exception {
+    // check for an expectedSlices new collection - we poll the state
+    long timeoutAt = System.currentTimeMillis() + 120000;
+    boolean success = false;
+    String checkResult = "Didnt get to perform a single check";
+    while (System.currentTimeMillis() < timeoutAt) {
+      checkResult = checkCollectionExpectations(collectionName,
+          numShardsNumReplicaList, nodesAllowedToRunShards);
+      if (checkResult == null) {
+        success = true;
+        break;
+      }
+      Thread.sleep(500);
+    }
+    if (!success) {
+      super.printLayout();
+      fail(checkResult);
+    }
+  }
+  
+  volatile CloudSolrServer commondCloudSolrServer;
+  protected CloudSolrServer getCommonCloudSolrServer() {
+    if (commondCloudSolrServer == null) {
+      synchronized(this) {
+        try {
+          commondCloudSolrServer = new CloudSolrServer(zkServer.getZkAddress());
+          commondCloudSolrServer.setDefaultCollection(DEFAULT_COLLECTION);
+          commondCloudSolrServer.connect();
+        } catch (MalformedURLException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+    return commondCloudSolrServer;
+  }
 
 }