You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/12/19 18:48:57 UTC

svn commit: r1552377 [9/15] - in /lucene/dev/branches/lucene5339: ./ dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/benchmark/src/ dev-tools/idea/lucene/demo/ dev-tools/idea/lucene/facet/ dev-tools/idea/solr/cont...

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java Thu Dec 19 17:48:47 2013
@@ -23,10 +23,10 @@ import org.apache.solr.client.solrj.impl
 import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.BeforeReconnect;
 import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DefaultConnectionStrategy;
 import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.DocRouter;
-import org.apache.solr.common.cloud.ImplicitDocRouter;
 import org.apache.solr.common.cloud.OnReconnect;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
@@ -172,11 +172,9 @@ public final class ZkController {
   private int clientTimeout;
 
   private volatile boolean isClosed;
-  
-  private UpdateShardHandler updateShardHandler;
 
   public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
-      String localHostContext, int leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
+      String localHostContext, int leaderVoteWait, boolean genericCoreNodeNames, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
       TimeoutException, IOException {
     if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null.");
     this.cc = cc;
@@ -187,8 +185,6 @@ public final class ZkController {
     // which means the default of "solr"
     localHostContext = trimLeadingAndTrailingSlashes(localHostContext);
     
-    updateShardHandler = new UpdateShardHandler(distribUpdateConnTimeout, distribUpdateSoTimeout);
-    
     this.zkServerAddress = zkServerAddress;
     this.localHostPort = locaHostPort;
     this.localHostContext = localHostContext;
@@ -203,46 +199,53 @@ public final class ZkController {
 
     this.leaderVoteWait = leaderVoteWait;
     this.clientTimeout = zkClientTimeout;
-    zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout, zkClientConnectTimeout,
+    zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout,
+        zkClientConnectTimeout, new DefaultConnectionStrategy(),
         // on reconnect, reload cloud info
         new OnReconnect() {
-
+          
           @Override
           public void command() {
             try {
               markAllAsNotLeader(registerOnReconnect);
               
-              // this is troublesome - we dont want to kill anything the old leader accepted
-              // though I guess sync will likely get those updates back? But only if
+              // this is troublesome - we dont want to kill anything the old
+              // leader accepted
+              // though I guess sync will likely get those updates back? But
+              // only if
               // he is involved in the sync, and he certainly may not be
-            //  ExecutorUtil.shutdownAndAwaitTermination(cc.getCmdDistribExecutor());
+              // ExecutorUtil.shutdownAndAwaitTermination(cc.getCmdDistribExecutor());
               // we need to create all of our lost watches
               
               // seems we dont need to do this again...
-              //Overseer.createClientNodes(zkClient, getNodeName());
+              // Overseer.createClientNodes(zkClient, getNodeName());
               ShardHandler shardHandler;
               String adminPath;
               shardHandler = cc.getShardHandlerFactory().getShardHandler();
               adminPath = cc.getAdminPath();
-
+              
               cc.cancelCoreRecoveries();
               
               registerAllCoresAsDown(registerOnReconnect, false);
-
-              ZkController.this.overseer = new Overseer(shardHandler, adminPath, zkStateReader);
-              ElectionContext context = new OverseerElectionContext(zkClient, overseer, getNodeName());
+              
+              ElectionContext context = new OverseerElectionContext(zkClient,
+                  overseer, getNodeName());
+              
               overseerElector.joinElection(context, true);
               zkStateReader.createClusterStateWatchersAndUpdate();
               
               // we have to register as live first to pick up docs in the buffer
               createEphemeralLiveNode();
               
-              List<CoreDescriptor> descriptors = registerOnReconnect.getCurrentDescriptors();
+              List<CoreDescriptor> descriptors = registerOnReconnect
+                  .getCurrentDescriptors();
               // re register all descriptors
-              if (descriptors  != null) {
+              if (descriptors != null) {
                 for (CoreDescriptor descriptor : descriptors) {
-                  // TODO: we need to think carefully about what happens when it was
-                  // a leader that was expired - as well as what to do about leaders/overseers
+                  // TODO: we need to think carefully about what happens when it
+                  // was
+                  // a leader that was expired - as well as what to do about
+                  // leaders/overseers
                   // with connection loss
                   try {
                     register(descriptor.getName(), descriptor, true, true);
@@ -251,7 +254,7 @@ public final class ZkController {
                   }
                 }
               }
-  
+              
             } catch (InterruptedException e) {
               // Restore the interrupted status
               Thread.currentThread().interrupt();
@@ -262,10 +265,18 @@ public final class ZkController {
               throw new ZooKeeperException(
                   SolrException.ErrorCode.SERVER_ERROR, "", e);
             }
-
           }
-
- 
+          
+        }, new BeforeReconnect() {
+          
+          @Override
+          public void command() {
+            try {
+              ZkController.this.overseer.close();
+            } catch (Exception e) {
+              log.error("Error trying to stop any Overseer threads", e);
+            }
+          }
         });
     
     this.overseerJobQueue = Overseer.getInQueue(zkClient);
@@ -396,13 +407,6 @@ public final class ZkController {
       log.error("Error closing zkClient", t);
     } 
     
-    if (updateShardHandler != null) {
-      try {
-        updateShardHandler.close();
-      } catch(Throwable t) {
-        log.error("Error closing updateShardHandler", t);
-      }
-    }
   }
 
   /**
@@ -718,35 +722,6 @@ public final class ZkController {
     return zkClient.exists(path, true);
   }
 
-  /**
-   * Returns config value
-   */
-  public String readConfigName(String collection) throws KeeperException,
-      InterruptedException {
-
-    String configName = null;
-
-    String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
-    if (log.isInfoEnabled()) {
-      log.info("Load collection config from:" + path);
-    }
-    byte[] data = zkClient.getData(path, null, null, true);
-    
-    if(data != null) {
-      ZkNodeProps props = ZkNodeProps.load(data);
-      configName = props.getStr(CONFIGNAME_PROP);
-    }
-    
-    if (configName != null && !zkClient.exists(CONFIGS_ZKNODE + "/" + configName, true)) {
-      log.error("Specified config does not exist in ZooKeeper:" + configName);
-      throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
-          "Specified config does not exist in ZooKeeper:" + configName);
-    }
-
-    return configName;
-  }
-
-
 
   /**
    * Register shard with ZooKeeper.
@@ -1358,30 +1333,31 @@ public final class ZkController {
   public void preRegister(CoreDescriptor cd ) {
 
     String coreNodeName = getCoreNodeName(cd);
-
-    // make sure the node name is set on the descriptor
-    if (cd.getCloudDescriptor().getCoreNodeName() == null) {
-      cd.getCloudDescriptor().setCoreNodeName(coreNodeName);
-    }
-
     // before becoming available, make sure we are not live and active
     // this also gets us our assigned shard id if it was not specified
     try {
-      if(cd.getCloudDescriptor().getCollectionName() !=null && cd.getCloudDescriptor().getCoreNodeName() != null ) {
+      CloudDescriptor cloudDesc = cd.getCloudDescriptor();
+      if(cd.getCloudDescriptor().getCollectionName() !=null && cloudDesc.getCoreNodeName() != null ) {
         //we were already registered
-        if(zkStateReader.getClusterState().hasCollection(cd.getCloudDescriptor().getCollectionName())){
-        DocCollection coll = zkStateReader.getClusterState().getCollection(cd.getCloudDescriptor().getCollectionName());
+        if(zkStateReader.getClusterState().hasCollection(cloudDesc.getCollectionName())){
+        DocCollection coll = zkStateReader.getClusterState().getCollection(cloudDesc.getCollectionName());
          if(!"true".equals(coll.getStr("autoCreated"))){
-           Slice slice = coll.getSlice(cd.getCloudDescriptor().getShardId());
+           Slice slice = coll.getSlice(cloudDesc.getShardId());
            if(slice != null){
-             if(slice.getReplica(cd.getCloudDescriptor().getCoreNodeName()) == null) {
+             if(slice.getReplica(cloudDesc.getCoreNodeName()) == null) {
                log.info("core_removed This core is removed from ZK");
-               throw new SolrException(ErrorCode.NOT_FOUND,coreNodeName +" is removed");
+               throw new SolrException(ErrorCode.NOT_FOUND,cloudDesc.getCoreNodeName() +" is removed");
              }
            }
          }
         }
       }
+
+      // make sure the node name is set on the descriptor
+      if (cloudDesc.getCoreNodeName() == null) {
+        cloudDesc.setCoreNodeName(coreNodeName);
+      }
+
       publish(cd, ZkStateReader.DOWN, false);
     } catch (KeeperException e) {
       log.error("", e);
@@ -1562,11 +1538,14 @@ public final class ZkController {
     return clientTimeout;
   }
 
-  // may return null if not in zk mode
-  public UpdateShardHandler getUpdateShardHandler() {
-    return updateShardHandler;
+  public Overseer getOverseer() {
+    return overseer;
   }
 
+  public LeaderElector getOverseerElector() {
+    return overseerElector;
+  }
+  
   /**
    * Returns the nodeName that should be used based on the specified properties.
    *

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java Thu Dec 19 17:48:47 2013
@@ -156,6 +156,14 @@ public abstract class ConfigSolr {
   public int getDistributedSocketTimeout() {
     return getInt(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, 0);
   }
+  
+  public int getMaxUpdateConnections() {
+    return getInt(CfgProp.SOLR_MAXUPDATECONNECTIONS, 10000);
+  }
+
+  public int getMaxUpdateConnectionsPerHost() {
+    return getInt(CfgProp.SOLR_MAXUPDATECONNECTIONSPERHOST, 100);
+  }
 
   public int getCoreLoadThreadCount() {
     return getInt(ConfigSolr.CfgProp.SOLR_CORELOADTHREADS, DEFAULT_CORE_LOAD_THREADS);
@@ -179,6 +187,14 @@ public abstract class ConfigSolr {
     return get(CfgProp.SOLR_ADMINHANDLER, "org.apache.solr.handler.admin.CoreAdminHandler");
   }
 
+  public String getCollectionsHandlerClass() {
+    return get(CfgProp.SOLR_COLLECTIONSHANDLER, "org.apache.solr.handler.admin.CollectionsHandler");
+  }
+
+  public String getInfoHandlerClass() {
+    return get(CfgProp.SOLR_INFOHANDLER, "org.apache.solr.handler.admin.InfoHandler");
+  }
+
   public boolean hasSchemaCache() {
     return getBool(ConfigSolr.CfgProp.SOLR_SHARESCHEMA, false);
   }
@@ -203,13 +219,17 @@ public abstract class ConfigSolr {
   // Ugly for now, but we'll at least be able to centralize all of the differences between 4x and 5x.
   protected static enum CfgProp {
     SOLR_ADMINHANDLER,
+    SOLR_COLLECTIONSHANDLER,
     SOLR_CORELOADTHREADS,
     SOLR_COREROOTDIRECTORY,
     SOLR_DISTRIBUPDATECONNTIMEOUT,
     SOLR_DISTRIBUPDATESOTIMEOUT,
+    SOLR_MAXUPDATECONNECTIONS,
+    SOLR_MAXUPDATECONNECTIONSPERHOST,
     SOLR_HOST,
     SOLR_HOSTCONTEXT,
     SOLR_HOSTPORT,
+    SOLR_INFOHANDLER,
     SOLR_LEADERVOTEWAIT,
     SOLR_LOGGING_CLASS,
     SOLR_LOGGING_ENABLED,

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java Thu Dec 19 17:48:47 2013
@@ -103,10 +103,14 @@ public class ConfigSolrXml extends Confi
   
   private void fillPropMap() {
     propMap.put(CfgProp.SOLR_ADMINHANDLER, doSub("solr/str[@name='adminHandler']"));
+    propMap.put(CfgProp.SOLR_COLLECTIONSHANDLER, doSub("solr/str[@name='collectionsHandler']"));
+    propMap.put(CfgProp.SOLR_INFOHANDLER, doSub("solr/str[@name='infoHandler']"));
     propMap.put(CfgProp.SOLR_CORELOADTHREADS, doSub("solr/int[@name='coreLoadThreads']"));
     propMap.put(CfgProp.SOLR_COREROOTDIRECTORY, doSub("solr/str[@name='coreRootDirectory']"));
     propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateConnTimeout']"));
     propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateSoTimeout']"));
+    propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONS, doSub("solr/solrcloud/int[@name='maxUpdateConnections']"));
+    propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONSPERHOST, doSub("solr/solrcloud/int[@name='maxUpdateConnectionsPerHost']"));
     propMap.put(CfgProp.SOLR_HOST, doSub("solr/solrcloud/str[@name='host']"));
     propMap.put(CfgProp.SOLR_HOSTCONTEXT, doSub("solr/solrcloud/str[@name='hostContext']"));
     propMap.put(CfgProp.SOLR_HOSTPORT, doSub("solr/solrcloud/int[@name='hostPort']"));

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Thu Dec 19 17:48:47 2013
@@ -141,10 +141,16 @@ public class ConfigSolrXmlOld extends Co
     
     propMap.put(CfgProp.SOLR_ADMINHANDLER,
         config.getVal("solr/cores/@adminHandler", false));
+    propMap.put(CfgProp.SOLR_COLLECTIONSHANDLER, config.getVal("solr/cores/@collectionsHandler", false));
+    propMap.put(CfgProp.SOLR_INFOHANDLER, config.getVal("solr/cores/@infoHandler", false));
     propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT,
         config.getVal("solr/cores/@distribUpdateConnTimeout", false));
     propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT,
         config.getVal("solr/cores/@distribUpdateSoTimeout", false));
+    propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONS,
+        config.getVal("solr/cores/@maxUpdateConnections", false));
+    propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONSPERHOST,
+        config.getVal("solr/cores/@maxUpdateConnectionsPerHost", false));
     propMap.put(CfgProp.SOLR_HOST, config.getVal("solr/cores/@host", false));
     propMap.put(CfgProp.SOLR_HOSTCONTEXT,
         config.getVal("solr/cores/@hostContext", false));

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java Thu Dec 19 17:48:47 2013
@@ -17,7 +17,26 @@
 
 package org.apache.solr.core;
 
-import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.collect.Maps;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.handler.admin.CoreAdminHandler;
+import org.apache.solr.handler.admin.InfoHandler;
+import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.logging.LogWatcher;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.IndexSchemaFactory;
+import org.apache.solr.update.UpdateShardHandler;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.FileUtils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.text.SimpleDateFormat;
@@ -40,27 +59,7 @@ import java.util.concurrent.ExecutorServ
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.cloud.ZkSolrResourceLoader;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.ZooKeeperException;
-import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.common.util.SolrjNamedThreadFactory;
-import org.apache.solr.handler.admin.CollectionsHandler;
-import org.apache.solr.handler.admin.CoreAdminHandler;
-import org.apache.solr.handler.admin.InfoHandler;
-import org.apache.solr.handler.component.ShardHandlerFactory;
-import org.apache.solr.logging.LogWatcher;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.IndexSchemaFactory;
-import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.util.FileUtils;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 
 /**
@@ -88,8 +87,7 @@ public class CoreContainer {
   protected ZkContainer zkSys = new ZkContainer();
   private ShardHandlerFactory shardHandlerFactory;
   
-  private ExecutorService updateExecutor = Executors.newCachedThreadPool(
-      new SolrjNamedThreadFactory("updateExecutor"));
+  private UpdateShardHandler updateShardHandler;
 
   protected LogWatcher logging = null;
 
@@ -102,6 +100,8 @@ public class CoreContainer {
   protected final CoresLocator coresLocator;
   
   private String hostName;
+  
+ // private ClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager();
 
   {
     log.info("New CoreContainer " + System.identityHashCode(this));
@@ -196,6 +196,8 @@ public class CoreContainer {
     }
 
     shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
+    
+    updateShardHandler = new UpdateShardHandler(cfg);
 
     solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
 
@@ -212,9 +214,9 @@ public class CoreContainer {
 
     zkSys.initZooKeeper(this, solrHome, cfg);
 
-    collectionsHandler = new CollectionsHandler(this);
-    infoHandler = new InfoHandler(this);
-    coreAdminHandler = createMultiCoreHandler(cfg.getCoreAdminHandlerClass());
+    collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
+    infoHandler        = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class);
+    coreAdminHandler   = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
 
     containerProperties = cfg.getSolrProperties("solr");
 
@@ -358,7 +360,6 @@ public class CoreContainer {
       cancelCoreRecoveries();
     }
 
-
     try {
       // First wake up the closer thread, it'll terminate almost immediately since it checks isShutDown.
       synchronized (solrCores.getModifyLock()) {
@@ -384,16 +385,20 @@ public class CoreContainer {
       }
 
     } finally {
-      if (shardHandlerFactory != null) {
-        shardHandlerFactory.close();
+      try {
+        if (shardHandlerFactory != null) {
+          shardHandlerFactory.close();
+        }
+      } finally {
+        try {
+          if (updateShardHandler != null) {
+            updateShardHandler.close();
+          }
+        } finally {
+          // we want to close zk stuff last
+          zkSys.close();
+        }
       }
-      
-      ExecutorUtil.shutdownAndAwaitTermination(updateExecutor);
-      
-      // we want to close zk stuff last
-
-      zkSys.close();
-
     }
     org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort
   }
@@ -676,7 +681,7 @@ public class CoreContainer {
             String collection = cd.getCloudDescriptor().getCollectionName();
             zkSys.getZkController().createCollectionZkNode(cd.getCloudDescriptor());
 
-            String zkConfigName = zkSys.getZkController().readConfigName(collection);
+            String zkConfigName = zkSys.getZkController().getZkStateReader().readConfigName(collection);
             if (zkConfigName == null) {
               log.error("Could not find config name for collection:" + collection);
               throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
@@ -726,7 +731,7 @@ public class CoreContainer {
     n1 = checkDefault(n1);
     solrCores.swap(n0, n1);
 
-    coresLocator.persist(this, solrCores.getCoreDescriptor(n0), solrCores.getCoreDescriptor(n1));
+    coresLocator.swap(this, solrCores.getCoreDescriptor(n0), solrCores.getCoreDescriptor(n1));
 
     log.info("swapped: "+n0 + " with " + n1);
   }
@@ -773,10 +778,10 @@ public class CoreContainer {
     return null;
   }
 
-  /** 
+  /**
    * Gets a core by name and increase its refcount.
    *
-   * @see SolrCore#close() 
+   * @see SolrCore#close()
    * @param name the core name
    * @return the core if found, null if a SolrCore by this name does not exist
    * @exception SolrException if a SolrCore with this name failed to be initialized
@@ -795,7 +800,7 @@ public class CoreContainer {
     // OK, it's not presently in any list, is it in the list of dynamic cores but not loaded yet? If so, load it.
     CoreDescriptor desc = solrCores.getDynamicDescriptor(name);
     if (desc == null) { //Nope, no transient core with this name
-      
+
       // if there was an error initalizing this core, throw a 500
       // error with the details for clients attempting to access it.
       Exception e = getCoreInitFailures().get(name);
@@ -826,7 +831,7 @@ public class CoreContainer {
       }
     } catch(Exception ex){
       // remains to be seen how transient cores and such
-      // will work in SolrCloud mode, but just to be future 
+      // will work in SolrCloud mode, but just to be future
       // proof...
       /*if (isZooKeeperAware()) {
         try {
@@ -846,34 +851,33 @@ public class CoreContainer {
     return core;
   }
 
-  // ---------------- Multicore self related methods ---------------
-  /** 
-   * Creates a CoreAdminHandler for this MultiCore.
-   * @return a CoreAdminHandler
-   */
-  protected CoreAdminHandler createMultiCoreHandler(final String adminHandlerClass) {
-    return loader.newAdminHandlerInstance(CoreContainer.this, adminHandlerClass);
+  // ---------------- CoreContainer request handlers --------------
+
+  protected <T> T createHandler(String handlerClass, Class<T> clazz) {
+    return loader.newInstance(handlerClass, clazz, null, new Class[] { CoreContainer.class }, new Object[] { this });
   }
 
   public CoreAdminHandler getMultiCoreHandler() {
     return coreAdminHandler;
   }
-  
+
   public CollectionsHandler getCollectionsHandler() {
     return collectionsHandler;
   }
-  
+
   public InfoHandler getInfoHandler() {
     return infoHandler;
   }
-  
+
+  // ---------------- Multicore self related methods ---------------
+
   /**
    * the default core name, or null if there is no default core name
    */
   public String getDefaultCoreName() {
     return cfg.getDefaultCoreName();
   }
-  
+
   // all of the following properties aren't synchronized
   // but this should be OK since they normally won't be changed rapidly
   @Deprecated
@@ -952,8 +956,8 @@ public class CoreContainer {
     return shardHandlerFactory;
   }
   
-  public ExecutorService getUpdateExecutor() {
-    return updateExecutor;
+  public UpdateShardHandler getUpdateShardHandler() {
+    return updateShardHandler;
   }
   
   // Just to tidy up the code where it did this in-line.
@@ -969,8 +973,6 @@ public class CoreContainer {
   String getCoreToOrigName(SolrCore core) {
     return solrCores.getCoreToOrigName(core);
   }
-  
-
 }
 
 class CloserThread extends Thread {

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java Thu Dec 19 17:48:47 2013
@@ -91,7 +91,11 @@ public class CorePropertiesLocator imple
 
   @Override
   public void delete(CoreContainer cc, CoreDescriptor... coreDescriptors) {
+    if (coreDescriptors == null) {
+      return;
+    }
     for (CoreDescriptor cd : coreDescriptors) {
+      if (cd == null) continue;
       File instanceDir = new File(cd.getInstanceDir());
       File propertiesFile = new File(instanceDir, PROPERTIES_FILENAME);
       propertiesFile.renameTo(new File(instanceDir, PROPERTIES_FILENAME + ".unloaded"));
@@ -107,6 +111,11 @@ public class CorePropertiesLocator imple
   }
 
   @Override
+  public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2) {
+    persist(cc, cd1, cd2);
+  }
+
+  @Override
   public List<CoreDescriptor> discover(CoreContainer cc) {
     logger.info("Looking for core definitions underneath {}", rootDirectory.getAbsolutePath());
     List<CoreDescriptor> cds = Lists.newArrayList();

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java Thu Dec 19 17:48:47 2013
@@ -56,6 +56,14 @@ public interface CoresLocator {
   public void rename(CoreContainer cc, CoreDescriptor oldCD, CoreDescriptor newCD);
 
   /**
+   * Swap two core definitions
+   * @param cc  the CoreContainer
+   * @param cd1 the core descriptor of the first core, after swapping
+   * @param cd2 the core descriptor of the second core, after swapping
+   */
+  public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2);
+
+  /**
    * Load all the CoreDescriptors from persistence store
    * @param cc the CoreContainer
    * @return a list of all CoreDescriptors found

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java Thu Dec 19 17:48:47 2013
@@ -320,7 +320,7 @@ public class JmxMonitoredMap<K, V> exten
         try {
           list.add(new Attribute(attribute, getAttribute(attribute)));
         } catch (Exception e) {
-          LOG.warn("Could not get attibute " + attribute);
+          LOG.warn("Could not get attribute " + attribute);
         }
       }
 

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java Thu Dec 19 17:48:47 2013
@@ -17,9 +17,42 @@
 
 package org.apache.solr.core;
 
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.xml.parsers.ParserConfigurationException;
+
 import org.apache.commons.io.IOUtils;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.IndexWriter;
@@ -40,6 +73,7 @@ import org.apache.solr.common.util.Simpl
 import org.apache.solr.core.DirectoryFactory.DirContext;
 import org.apache.solr.handler.SnapPuller;
 import org.apache.solr.handler.admin.ShowFileRequestHandler;
+import org.apache.solr.handler.component.AnalyticsComponent;
 import org.apache.solr.handler.component.DebugComponent;
 import org.apache.solr.handler.component.FacetComponent;
 import org.apache.solr.handler.component.HighlightComponent;
@@ -93,39 +127,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
 
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Writer;
-import java.lang.reflect.Constructor;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.StringTokenizer;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-
 
 /**
  *
@@ -769,7 +770,7 @@ public final class SolrCore implements S
       updateProcessorChains = loadUpdateProcessorChains();
       reqHandlers = new RequestHandlers(this);
       reqHandlers.initHandlersFromConfig(solrConfig);
-      
+
       // Handle things that should eventually go away
       initDeprecatedSupport();
       
@@ -853,13 +854,21 @@ public final class SolrCore implements S
     
     CoreContainer cc = cd.getCoreContainer();
 
-    if (cc != null && cc.isZooKeeperAware() && Slice.CONSTRUCTION.equals(cd.getCloudDescriptor().getShardState())) {
-      // set update log to buffer before publishing the core
-      getUpdateHandler().getUpdateLog().bufferUpdates();
-      
-      cd.getCloudDescriptor().setShardState(null);
-      cd.getCloudDescriptor().setShardRange(null);
-      cd.getCloudDescriptor().setShardParent(null);
+    if (cc != null && cc.isZooKeeperAware()) {
+      SolrRequestHandler realtimeGetHandler = reqHandlers.get("/get");
+      if (realtimeGetHandler == null) {
+        log.warn("WARNING: RealTimeGetHandler is not registered at /get. " +
+            "SolrCloud will always use full index replication instead of the more efficient PeerSync method.");
+      }
+
+      if (Slice.CONSTRUCTION.equals(cd.getCloudDescriptor().getShardState())) {
+        // set update log to buffer before publishing the core
+        getUpdateHandler().getUpdateLog().bufferUpdates();
+
+        cd.getCloudDescriptor().setShardState(null);
+        cd.getCloudDescriptor().setShardRange(null);
+        cd.getCloudDescriptor().setShardParent(null);
+      }
     }
     // For debugging   
 //    numOpens.incrementAndGet();
@@ -1197,6 +1206,7 @@ public final class SolrCore implements S
     addIfNotPresent(components,StatsComponent.COMPONENT_NAME,StatsComponent.class);
     addIfNotPresent(components,DebugComponent.COMPONENT_NAME,DebugComponent.class);
     addIfNotPresent(components,RealTimeGetComponent.COMPONENT_NAME,RealTimeGetComponent.class);
+    addIfNotPresent(components,AnalyticsComponent.COMPONENT_NAME,AnalyticsComponent.class);
     return components;
   }
   private <T> void addIfNotPresent(Map<String ,T> registry, String name, Class<? extends  T> c){

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Thu Dec 19 17:48:47 2013
@@ -79,7 +79,7 @@ public class SolrResourceLoader implemen
 
   static final String project = "solr";
   static final String base = "org.apache" + "." + project;
-  static final String[] packages = {"","analysis.","schema.","handler.","search.","update.","core.","response.","request.","update.processor.","util.", "spelling.", "handler.component.", "handler.dataimport." };
+  static final String[] packages = {"","analysis.","schema.","handler.","search.","update.","core.","response.","request.","update.processor.","util.", "spelling.", "handler.component.", "handler.dataimport.", "spelling.suggest.", "spelling.suggest.fst." };
 
   protected URLClassLoader classLoader;
   private final String instanceDir;

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java Thu Dec 19 17:48:47 2013
@@ -19,7 +19,6 @@ package org.apache.solr.core;
 
 import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableList;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
@@ -32,11 +31,9 @@ import java.io.OutputStreamWriter;
 import java.io.Writer;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -191,6 +188,11 @@ public class SolrXMLCoresLocator impleme
   }
 
   @Override
+  public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2) {
+    this.persist(cc);
+  }
+
+  @Override
   public List<CoreDescriptor> discover(CoreContainer cc) {
 
     ImmutableList.Builder<CoreDescriptor> listBuilder = ImmutableList.builder();

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java Thu Dec 19 17:48:47 2013
@@ -51,8 +51,6 @@ public class ZkContainer {
   private String host;
   private int leaderVoteWait;
   private Boolean genericCoreNodeNames;
-  private int distribUpdateConnTimeout;
-  private int distribUpdateSoTimeout;
   
   public ZkContainer() {
     
@@ -67,13 +65,11 @@ public class ZkContainer {
 
     initZooKeeper(cc, solrHome,
         config.getZkHost(), config.getZkClientTimeout(), config.getZkHostPort(), config.getZkHostContext(),
-        config.getHost(), config.getLeaderVoteWait(), config.getGenericCoreNodeNames(),
-        config.getDistributedConnectionTimeout(), config.getDistributedSocketTimeout());
+        config.getHost(), config.getLeaderVoteWait(), config.getGenericCoreNodeNames());
   }
   
   public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort,
-                            String hostContext, String host, int leaderVoteWait, boolean genericCoreNodeNames,
-                            int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
+                            String hostContext, String host, int leaderVoteWait, boolean genericCoreNodeNames) {
     ZkController zkController = null;
     
     // if zkHost sys property is not set, we are not using ZooKeeper
@@ -92,8 +88,6 @@ public class ZkContainer {
     this.host = host;
     this.leaderVoteWait = leaderVoteWait;
     this.genericCoreNodeNames = genericCoreNodeNames;
-    this.distribUpdateConnTimeout = distribUpdateConnTimeout;
-    this.distribUpdateSoTimeout = distribUpdateSoTimeout;
     
     if (zkRun == null && zookeeperHost == null)
         return;  // not in zk mode
@@ -125,7 +119,7 @@ public class ZkContainer {
       }
     }
 
-    int zkClientConnectTimeout = 15000;
+    int zkClientConnectTimeout = 30000;
 
     if (zookeeperHost != null) {
 
@@ -147,7 +141,7 @@ public class ZkContainer {
         }
         zkController = new ZkController(cc, zookeeperHost, zkClientTimeout,
             zkClientConnectTimeout, host, hostPort, hostContext,
-            leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout,
+            leaderVoteWait, genericCoreNodeNames,
             new CurrentCoreDescriptorProvider() {
 
               @Override
@@ -220,7 +214,7 @@ public class ZkContainer {
       String collection = dcore.getCloudDescriptor().getCollectionName();
       zkController.createCollectionZkNode(dcore.getCloudDescriptor());
       
-      zkConfigName = zkController.readConfigName(collection);
+      zkConfigName = zkController.getZkStateReader().readConfigName(collection);
       if (zkConfigName == null) {
         log.error("Could not find config name for collection:" + collection);
         throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Thu Dec 19 17:48:47 2013
@@ -16,6 +16,7 @@
  */
 package org.apache.solr.handler;
 
+import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
 import static org.apache.solr.handler.ReplicationHandler.ALIAS;
 import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
 import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
@@ -47,6 +48,7 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -55,6 +57,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -75,9 +78,6 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-
-import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
-
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -104,6 +104,7 @@ import org.apache.solr.util.FileUtils;
 import org.apache.solr.util.PropertiesInputStream;
 import org.apache.solr.util.PropertiesOutputStream;
 import org.apache.solr.util.RefCounted;
+import org.eclipse.jetty.util.log.Log;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -162,25 +163,18 @@ public class SnapPuller {
    */
   private AtomicBoolean pollDisabled = new AtomicBoolean(false);
 
-  // HttpClient shared by all cores (used if timeout is not specified for a core)
-  private static HttpClient client;
-  // HttpClient for this instance if connectionTimeout or readTimeout has been specified
   private final HttpClient myHttpClient;
 
-  private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) {
-    if (connTimeout == null && readTimeout == null && client != null)  return client;
+  private static HttpClient createHttpClient(SolrCore core, String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) {
     final ModifiableSolrParams httpClientParams = new ModifiableSolrParams();
     httpClientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connTimeout != null ? connTimeout : "5000");
     httpClientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, readTimeout != null ? readTimeout : "20000");
     httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser);
     httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword);
     httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression);
-    // Keeping a very high number so that if you have a large number of cores
-    // no requests are kept waiting for an idle connection.
-    httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000);
-    httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 10000);
-    HttpClient httpClient = HttpClientUtil.createClient(httpClientParams);
-    if (client == null && connTimeout == null && readTimeout == null) client = httpClient;
+
+    HttpClient httpClient = HttpClientUtil.createClient(httpClientParams, core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getConnectionManager());
+
     return httpClient;
   }
 
@@ -207,7 +201,7 @@ public class SnapPuller {
     String readTimeout = (String) initArgs.get(HttpClientUtil.PROP_SO_TIMEOUT);
     String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
     String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
-    myHttpClient = createHttpClient(connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, useExternal);
+    myHttpClient = createHttpClient(solrCore, connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, useExternal);
     if (pollInterval != null && pollInterval > 0) {
       startExecutorService();
     } else {
@@ -388,8 +382,8 @@ public class SnapPuller {
       fsyncService = Executors.newSingleThreadExecutor(new DefaultSolrThreadFactory("fsyncService"));
       // use a synchronized list because the list is read by other threads (to show details)
       filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
-      // if the generateion of master is older than that of the slave , it means they are not compatible to be copied
-      // then a new index direcory to be created and all the files need to be copied
+      // if the generation of master is older than that of the slave , it means they are not compatible to be copied
+      // then a new index directory to be created and all the files need to be copied
       boolean isFullCopyNeeded = IndexDeletionPolicyWrapper
           .getCommitTimestamp(commit) >= latestVersion
           || commit.getGeneration() >= latestGeneration || forceReplication;
@@ -408,59 +402,76 @@ public class SnapPuller {
         if (isIndexStale(indexDir)) {
           isFullCopyNeeded = true;
         }
-        LOG.info("Starting download to " + tmpIndexDir + " fullCopy=" + isFullCopyNeeded);
-        successfulInstall = false;
         
-        downloadIndexFiles(isFullCopyNeeded, tmpIndexDir, latestGeneration);
-        LOG.info("Total time taken for download : " + ((System.currentTimeMillis() - replicationStartTime) / 1000) + " secs");
-        Collection<Map<String, Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
-        if (!modifiedConfFiles.isEmpty()) {
-          downloadConfFiles(confFilesToDownload, latestGeneration);
-          if (isFullCopyNeeded) {
-            successfulInstall = modifyIndexProps(tmpIdxDirName);
-            deleteTmpIdxDir  =  false;
-          } else {
-            solrCore.getUpdateHandler().getSolrCoreState()
-                .closeIndexWriter(core, true);
-            try {
+        if (!isFullCopyNeeded) {
+          // rollback - and do it before we download any files
+          // so we don't remove files we thought we didn't need
+          // to download later
+          solrCore.getUpdateHandler().getSolrCoreState()
+          .closeIndexWriter(core, true);
+        }
+        
+        boolean reloadCore = false;
+        
+        try {
+          LOG.info("Starting download to " + tmpIndexDir + " fullCopy="
+              + isFullCopyNeeded);
+          successfulInstall = false;
+          
+          downloadIndexFiles(isFullCopyNeeded, indexDir, tmpIndexDir,
+              latestGeneration);
+          LOG.info("Total time taken for download : "
+              + ((System.currentTimeMillis() - replicationStartTime) / 1000)
+              + " secs");
+          Collection<Map<String,Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
+          if (!modifiedConfFiles.isEmpty()) {
+            downloadConfFiles(confFilesToDownload, latestGeneration);
+            if (isFullCopyNeeded) {
+              successfulInstall = modifyIndexProps(tmpIdxDirName);
+              deleteTmpIdxDir = false;
+            } else {
               successfulInstall = moveIndexFiles(tmpIndexDir, indexDir);
-            } finally {
-              solrCore.getUpdateHandler().getSolrCoreState()
-                  .openIndexWriter(core);
             }
-          }
-          if (successfulInstall) {
-            if (isFullCopyNeeded) {
-              // let the system know we are changing dir's and the old one
-              // may be closed
-              if (indexDir != null) {
-                LOG.info("removing old index directory " + indexDir);
-                core.getDirectoryFactory().doneWithDirectory(indexDir);
-                core.getDirectoryFactory().remove(indexDir);
+            if (successfulInstall) {
+              if (isFullCopyNeeded) {
+                // let the system know we are changing dir's and the old one
+                // may be closed
+                if (indexDir != null) {
+                  LOG.info("removing old index directory " + indexDir);
+                  core.getDirectoryFactory().doneWithDirectory(indexDir);
+                  core.getDirectoryFactory().remove(indexDir);
+                }
               }
+              
+              LOG.info("Configuration files are modified, core will be reloaded");
+              logReplicationTimeAndConfFiles(modifiedConfFiles,
+                  successfulInstall);// write to a file time of replication and
+                                     // conf files.
+              reloadCore = true;
             }
-            
-            LOG.info("Configuration files are modified, core will be reloaded");
-            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);//write to a file time of replication and conf files.
-            reloadCore();
-          }
-        } else {
-          terminateAndWaitFsyncService();
-          if (isFullCopyNeeded) {
-            successfulInstall = modifyIndexProps(tmpIdxDirName);
-            deleteTmpIdxDir =  false;
           } else {
-            solrCore.getUpdateHandler().getSolrCoreState().closeIndexWriter(core, true);
-            try {
+            terminateAndWaitFsyncService();
+            if (isFullCopyNeeded) {
+              successfulInstall = modifyIndexProps(tmpIdxDirName);
+              deleteTmpIdxDir = false;
+            } else {
               successfulInstall = moveIndexFiles(tmpIndexDir, indexDir);
-            } finally {
-              solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(core);
+            }
+            if (successfulInstall) {
+              logReplicationTimeAndConfFiles(modifiedConfFiles,
+                  successfulInstall);
             }
           }
-          if (successfulInstall) {
-            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);
+        } finally {
+          if (!isFullCopyNeeded) {
+            solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(core);
           }
         }
+        
+        // we must reload the core after we open the IW back up
+        if (reloadCore) {
+          reloadCore();
+        }
 
         if (successfulInstall) {
           if (isFullCopyNeeded) {
@@ -690,6 +701,7 @@ public class SnapPuller {
   }
 
   private void reloadCore() {
+    final CountDownLatch latch = new CountDownLatch(1);
     new Thread() {
       @Override
       public void run() {
@@ -697,9 +709,17 @@ public class SnapPuller {
           solrCore.getCoreDescriptor().getCoreContainer().reload(solrCore.getName());
         } catch (Exception e) {
           LOG.error("Could not reload core ", e);
+        } finally {
+          latch.countDown();
         }
       }
     }.start();
+    try {
+      latch.await();
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new RuntimeException("Interrupted while waiting for core reload to finish", e);
+    }
   }
 
   private void downloadConfFiles(List<Map<String, Object>> confFilesToDownload, long latestGeneration) throws Exception {
@@ -732,29 +752,28 @@ public class SnapPuller {
    * Download the index files. If a new index is needed, download all the files.
    *
    * @param downloadCompleteIndex is it a fresh index copy
-   * @param tmpIndexDir               the directory to which files need to be downloadeed to
+   * @param tmpIndexDir              the directory to which files need to be downloadeed to
+   * @param indexDir                 the indexDir to be merged to
    * @param latestGeneration         the version number
    */
   private void downloadIndexFiles(boolean downloadCompleteIndex,
-      Directory tmpIndexDir, long latestGeneration) throws Exception {
-    String indexDir = solrCore.getIndexDir();
-    
-    // it's okay to use null for lock factory since we know this dir will exist
-    Directory dir = solrCore.getDirectoryFactory().get(indexDir, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
-    try {
-      for (Map<String,Object> file : filesToDownload) {
-        if (!dir.fileExists((String) file.get(NAME)) || downloadCompleteIndex) {
-          dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file,
-              (String) file.get(NAME), false, latestGeneration);
-          currentFile = file;
-          dirFileFetcher.fetchFile();
-          filesDownloaded.add(new HashMap<String,Object>(file));
-        } else {
-          LOG.info("Skipping download for " + file.get(NAME) + " because it already exists");
-        }
+      Directory indexDir, Directory tmpIndexDir, long latestGeneration)
+      throws Exception {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Download files to dir: " + Arrays.asList(indexDir.listAll()));
+    }
+    for (Map<String,Object> file : filesToDownload) {
+      if (!indexDir.fileExists((String) file.get(NAME))
+          || downloadCompleteIndex) {
+        dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file,
+            (String) file.get(NAME), false, latestGeneration);
+        currentFile = file;
+        dirFileFetcher.fetchFile();
+        filesDownloaded.add(new HashMap<String,Object>(file));
+      } else {
+        LOG.info("Skipping download for " + file.get(NAME)
+            + " because it already exists");
       }
-    } finally {
-      solrCore.getDirectoryFactory().release(dir);
     }
   }
 
@@ -769,6 +788,8 @@ public class SnapPuller {
     for (Map<String, Object> file : filesToDownload) {
       if (dir.fileExists((String) file.get(NAME))
               && dir.fileLength((String) file.get(NAME)) != (Long) file.get(SIZE)) {
+        LOG.warn("File " + file.get(NAME) + " expected to be " + file.get(SIZE)
+            + " while it is " + dir.fileLength((String) file.get(NAME)));
         // file exists and size is different, therefore we must assume
         // corrupted index
         return true;
@@ -782,6 +803,7 @@ public class SnapPuller {
    * <p/>
    */
   private boolean moveAFile(Directory tmpIdxDir, Directory indexDir, String fname, List<String> copiedfiles) {
+    LOG.debug("Moving file: {}", fname);
     boolean success = false;
     try {
       if (indexDir.fileExists(fname)) {
@@ -805,6 +827,14 @@ public class SnapPuller {
    * Copy all index files from the temp index dir to the actual index. The segments_N file is copied last.
    */
   private boolean moveIndexFiles(Directory tmpIdxDir, Directory indexDir) {
+    if (LOG.isDebugEnabled()) {
+      try {
+        LOG.info("From dir files:" + Arrays.asList(tmpIdxDir.listAll()));
+        LOG.info("To dir files:" + Arrays.asList(indexDir.listAll()));
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
     String segmentsFile = null;
     List<String> movedfiles = new ArrayList<String>();
     for (Map<String, Object> f : filesDownloaded) {

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java Thu Dec 19 17:48:47 2013
@@ -75,7 +75,7 @@ public class UpdateRequestHandler extend
         type = stream.getContentType();
       }
       if( type == null ) { // Normal requests will not get here.
-        throw new SolrException(ErrorCode.BAD_REQUEST, "Missing ContentType");
+        throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Missing ContentType");
       }
       int idx = type.indexOf(';');
       if(idx>0) {
@@ -83,7 +83,7 @@ public class UpdateRequestHandler extend
       }
       ContentStreamLoader loader = loaders.get(type);
       if(loader==null) {
-        throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported ContentType: "
+        throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Unsupported ContentType: "
             +type+ "  Not in: "+loaders.keySet());
       }
       if(loader.getDefaultWT()!=null) {

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Thu Dec 19 17:48:47 2013
@@ -318,6 +318,7 @@ public class CollectionsHandler extends 
         SHARDS_PROP,
         "router.");
 
+    copyPropertiesIfNotNull(req.getParams(), props);
 
     ZkNodeProps m = new ZkNodeProps(props);
     handleResponse(OverseerCollectionProcessor.CREATECOLLECTION, m, rsp);
@@ -342,6 +343,7 @@ public class CollectionsHandler extends 
 
     Map<String, Object> map = makeMap(QUEUE_OPERATION, CREATESHARD);
     copyIfNotNull(req.getParams(),map,COLLECTION_PROP, SHARD_ID_PROP, REPLICATION_FACTOR,CREATE_NODE_SET);
+    copyPropertiesIfNotNull(req.getParams(), map);
     ZkNodeProps m = new ZkNodeProps(map);
     handleResponse(CREATESHARD, m, rsp);
   }
@@ -372,7 +374,18 @@ public class CollectionsHandler extends 
     }
 
   }
-  
+
+  private void copyPropertiesIfNotNull(SolrParams params, Map<String, Object> props) {
+    Iterator<String> iter =  params.getParameterNamesIterator();
+    while (iter.hasNext()) {
+      String param = iter.next();
+      if (param.startsWith(OverseerCollectionProcessor.COLL_PROP_PREFIX)) {
+        props.put(param, params.get(param));
+      }
+    }
+  }
+
+
   private void handleDeleteShardAction(SolrQueryRequest req,
       SolrQueryResponse rsp) throws InterruptedException, KeeperException {
     log.info("Deleting Shard : " + req.getParamString());
@@ -420,6 +433,7 @@ public class CollectionsHandler extends 
     if (rangesStr != null)  {
       props.put(CoreAdminParams.RANGES, rangesStr);
     }
+    copyPropertiesIfNotNull(req.getParams(), props);
 
     ZkNodeProps m = new ZkNodeProps(props);
 

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Thu Dec 19 17:48:47 2013
@@ -596,6 +596,15 @@ public class CoreAdminHandler extends Re
             "No such core exists '" + cname + "'");
       } else {
         if (coreContainer.getZkController() != null) {
+          // we are unloading, cancel any ongoing recovery
+          // so there are no races to publish state
+          // we will try to cancel again later before close
+          if (core != null) {
+            if (coreContainer.getZkController() != null) {
+              core.getSolrCoreState().cancelRecovery();
+            }
+          }
+          
           log.info("Unregistering core " + core.getName() + " from cloudstate.");
           try {
             coreContainer.getZkController().unregister(cname,
@@ -796,7 +805,7 @@ public class CoreAdminHandler extends Re
     try {
       core = coreContainer.getCore(cname);
       if (core != null) {
-        syncStrategy = new SyncStrategy();
+        syncStrategy = new SyncStrategy(core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler());
         
         Map<String,Object> props = new HashMap<String,Object>();
         props.put(ZkStateReader.BASE_URL_PROP, zkController.getBaseUrl());

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java Thu Dec 19 17:48:47 2013
@@ -23,6 +23,7 @@ import org.apache.solr.common.SolrExcept
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
@@ -33,7 +34,11 @@ import org.apache.solr.handler.RequestHa
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.RawResponseWriter;
 import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.ManagedIndexSchema;
 import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -60,10 +65,15 @@ import java.util.Set;
  *   &lt;/lst&gt;
  *   &lt;lst name="invariants"&gt;
  *    &lt;str name="hidden"&gt;synonyms.txt&lt;/str&gt; 
- *    &lt;str name="hidden"&gt;anotherfile.txt&lt;/str&gt; 
+ *    &lt;str name="hidden"&gt;anotherfile.txt&lt;/str&gt;
+ *    &lt;str name="hidden"&gt;*&lt;/str&gt;
  *   &lt;/lst&gt;
  * &lt;/requestHandler&gt;
  * </pre>
+ *
+ * At present, there is only explicit file names (including path) or the glob '*' are supported. Variants like '*.xml'
+ * are NOT supported.ere
+ *
  * <p>
  * The ShowFileRequestHandler uses the {@link RawResponseWriter} (wt=raw) to return
  * file contents.  If you need to use a different writer, you will need to change 
@@ -75,7 +85,7 @@ import java.util.Set;
  * <pre>
  *   http://localhost:8983/solr/admin/file?file=schema.xml&contentType=text/plain
  * </pre>
- * 
+ *
  *
  * @since solr 1.3
  */
@@ -83,9 +93,13 @@ public class ShowFileRequestHandler exte
 {
   public static final String HIDDEN = "hidden";
   public static final String USE_CONTENT_TYPE = "contentType";
-  
+
   protected Set<String> hiddenFiles;
-  
+
+  protected static final Logger log = LoggerFactory
+      .getLogger(ShowFileRequestHandler.class);
+
+
   public ShowFileRequestHandler()
   {
     super();
@@ -94,27 +108,28 @@ public class ShowFileRequestHandler exte
   @Override
   public void init(NamedList args) {
     super.init( args );
+    hiddenFiles = initHidden(invariants);
+  }
+
+  public static Set<String> initHidden(SolrParams invariants) {
 
+    Set<String> hiddenRet = new HashSet<String>();
     // Build a list of hidden files
-    hiddenFiles = new HashSet<String>();
-    if( invariants != null ) {
-      String[] hidden = invariants.getParams( HIDDEN );
-      if( hidden != null ) {
-        for( String s : hidden ) {
-          hiddenFiles.add( s.toUpperCase(Locale.ROOT) );
+    if (invariants != null) {
+      String[] hidden = invariants.getParams(HIDDEN);
+      if (hidden != null) {
+        for (String s : hidden) {
+          hiddenRet.add(s.toUpperCase(Locale.ROOT));
         }
       }
     }
+    return hiddenRet;
   }
-  
-  public Set<String> getHiddenFiles()
-  {
-    return hiddenFiles;
-  }
-  
+
   @Override
-  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, KeeperException, InterruptedException 
-  {
+  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
+      throws InterruptedException, KeeperException, IOException {
+
     CoreContainer coreContainer = req.getCore().getCoreDescriptor().getCoreContainer();
     if (coreContainer.isZooKeeperAware()) {
       showFromZooKeeper(req, rsp, coreContainer);
@@ -123,58 +138,32 @@ public class ShowFileRequestHandler exte
     }
   }
 
+  // Get a list of files from ZooKeeper for from the path in the file= parameter.
   private void showFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp,
       CoreContainer coreContainer) throws KeeperException,
       InterruptedException, UnsupportedEncodingException {
-    String adminFile = null;
-    SolrCore core = req.getCore();
+
     SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
-    final ZkSolrResourceLoader loader = (ZkSolrResourceLoader) core
-        .getResourceLoader();
-    String confPath = loader.getCollectionZkPath();
-    
-    String fname = req.getParams().get("file", null);
-    if (fname == null) {
-      adminFile = confPath;
-    } else {
-      fname = fname.replace('\\', '/'); // normalize slashes
-      if (hiddenFiles.contains(fname.toUpperCase(Locale.ROOT))) {
-        rsp.setException(new SolrException(ErrorCode.FORBIDDEN, "Can not access: " + fname));
-        return;
-      }
-      if (fname.indexOf("..") >= 0) {
-        rsp.setException(new SolrException(ErrorCode.FORBIDDEN, "Invalid path: " + fname));
-        return;
-      }
-      if (fname.startsWith("/")) { // Only files relative to conf are valid
-        fname = fname.substring(1);
-      }
-      adminFile = confPath + "/" + fname;
-    }
-    
-    // Make sure the file exists, is readable and is not a hidden file
-    if (!zkClient.exists(adminFile, true)) {
-      rsp.setException(new SolrException(ErrorCode.NOT_FOUND, "Can not find: "
-                                         + adminFile));
+
+    String adminFile = getAdminFileFromZooKeeper(req, rsp, zkClient, hiddenFiles);
+
+    if (adminFile == null) {
       return;
     }
-    
+
     // Show a directory listing
     List<String> children = zkClient.getChildren(adminFile, null, true);
     if (children.size() > 0) {
       
       NamedList<SimpleOrderedMap<Object>> files = new SimpleOrderedMap<SimpleOrderedMap<Object>>();
       for (String f : children) {
-        if (hiddenFiles.contains(f.toUpperCase(Locale.ROOT))) {
-          continue; // don't show 'hidden' files
-        }
-        if (f.startsWith(".")) {
-          continue; // skip hidden system files...
+        if (isHiddenFile(req, rsp, f, false, hiddenFiles)) {
+          continue;
         }
-        
+
         SimpleOrderedMap<Object> fileInfo = new SimpleOrderedMap<Object>();
         files.add(f, fileInfo);
-        List<String> fchildren = zkClient.getChildren(adminFile, null, true);
+        List<String> fchildren = zkClient.getChildren(adminFile + "/" + f, null, true);
         if (fchildren.size() > 0) {
           fileInfo.add("directory", true);
         } else {
@@ -199,45 +188,24 @@ public class ShowFileRequestHandler exte
     rsp.setHttpCaching(false);
   }
 
+  // Return the file indicated (or the directory listing) from the local file system.
   private void showFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) {
-    File adminFile = null;
-    
-    final SolrResourceLoader loader = req.getCore().getResourceLoader();
-    File configdir = new File( loader.getConfigDir() );
-    if (!configdir.exists()) {
-      // TODO: maybe we should just open it this way to start with?
-      try {
-        configdir = new File( loader.getClassLoader().getResource(loader.getConfigDir()).toURI() );
-      } catch (URISyntaxException e) {
-        rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Can not access configuration directory!", e));
-        return;
-      }
-    }
-    String fname = req.getParams().get("file", null);
-    if( fname == null ) {
-      adminFile = configdir;
-    }
-    else {
-      fname = fname.replace( '\\', '/' ); // normalize slashes
-      if( hiddenFiles.contains( fname.toUpperCase(Locale.ROOT) ) ) {
-        rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Can not access: "+fname ));
-        return;
-      }
-      if( fname.indexOf( ".." ) >= 0 ) {
-        rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Invalid path: "+fname ));
-        return;
-      }
-      adminFile = new File( configdir, fname );
+    File adminFile = getAdminFileFromFileSystem(req, rsp, hiddenFiles);
+
+    if (adminFile == null) { // exception already recorded
+      return;
     }
-    
+
     // Make sure the file exists, is readable and is not a hidden file
     if( !adminFile.exists() ) {
+      log.error("Can not find: "+adminFile.getName() + " ["+adminFile.getAbsolutePath()+"]");
       rsp.setException(new SolrException
                        ( ErrorCode.NOT_FOUND, "Can not find: "+adminFile.getName() 
                          + " ["+adminFile.getAbsolutePath()+"]" ));
       return;
     }
     if( !adminFile.canRead() || adminFile.isHidden() ) {
+      log.error("Can not show: "+adminFile.getName() + " ["+adminFile.getAbsolutePath()+"]");
       rsp.setException(new SolrException
                        ( ErrorCode.NOT_FOUND, "Can not show: "+adminFile.getName() 
                          + " ["+adminFile.getAbsolutePath()+"]" ));
@@ -246,19 +214,17 @@ public class ShowFileRequestHandler exte
     
     // Show a directory listing
     if( adminFile.isDirectory() ) {
-      
-      int basePath = configdir.getAbsolutePath().length() + 1;
+      // it's really a directory, just go for it.
+      int basePath = adminFile.getAbsolutePath().length() + 1;
       NamedList<SimpleOrderedMap<Object>> files = new SimpleOrderedMap<SimpleOrderedMap<Object>>();
       for( File f : adminFile.listFiles() ) {
         String path = f.getAbsolutePath().substring( basePath );
         path = path.replace( '\\', '/' ); // normalize slashes
-        if( hiddenFiles.contains( path.toUpperCase(Locale.ROOT) ) ) {
-          continue; // don't show 'hidden' files
-        }
-        if( f.isHidden() || f.getName().startsWith( "." ) ) {
-          continue; // skip hidden system files...
+
+        if (isHiddenFile(req, rsp, f.getName().replace('\\', '/'), false, hiddenFiles)) {
+          continue;
         }
-        
+
         SimpleOrderedMap<Object> fileInfo = new SimpleOrderedMap<Object>();
         files.add( path, fileInfo );
         if( f.isDirectory() ) {
@@ -270,7 +236,7 @@ public class ShowFileRequestHandler exte
         }
         fileInfo.add( "modified", new Date( f.lastModified() ) );
       }
-      rsp.add( "files", files );
+      rsp.add("files", files);
     }
     else {
       // Include the file contents
@@ -280,19 +246,139 @@ public class ShowFileRequestHandler exte
       req.setParams(params);
 
       ContentStreamBase content = new ContentStreamBase.FileStream( adminFile );
-      content.setContentType( req.getParams().get( USE_CONTENT_TYPE ) );
+      content.setContentType(req.getParams().get(USE_CONTENT_TYPE));
 
       rsp.add(RawResponseWriter.CONTENT, content);
     }
     rsp.setHttpCaching(false);
   }
-  
-  
+
+  //////////////////////// Static methods //////////////////////////////
+
+  public static boolean isHiddenFile(SolrQueryRequest req, SolrQueryResponse rsp, String fnameIn, boolean reportError,
+                                     Set<String> hiddenFiles) {
+    String fname = fnameIn.toUpperCase(Locale.ROOT);
+    if (hiddenFiles.contains(fname) || hiddenFiles.contains("*")) {
+      if (reportError) {
+        log.error("Cannot access " + fname);
+        rsp.setException(new SolrException(SolrException.ErrorCode.FORBIDDEN, "Can not access: " + fnameIn));
+      }
+      return true;
+    }
+
+    // This is slightly off, a valid path is something like ./schema.xml. I don't think it's worth the effort though
+    // to fix it to handle all possibilities though.
+    if (fname.indexOf("..") >= 0 || fname.startsWith(".")) {
+      if (reportError) {
+        log.error("Invalid path: " + fname);
+        rsp.setException(new SolrException(SolrException.ErrorCode.FORBIDDEN, "Invalid path: " + fnameIn));
+      }
+      return true;
+    }
+
+    // Make sure that if the schema is managed, we don't allow editing. Don't really want to put
+    // this in the init since we're not entirely sure when the managed schema will get initialized relative to this
+    // handler.
+    SolrCore core = req.getCore();
+    IndexSchema schema = core.getLatestSchema();
+    if (schema instanceof ManagedIndexSchema) {
+      String managed = schema.getResourceName();
+
+      if (fname.equalsIgnoreCase(managed)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // Refactored to be usable from multiple methods. Gets the path of the requested file from ZK.
+  // Returns null if the file is not found.
+  //
+  // Assumes that the file is in a parameter called "file".
+
+  public static String getAdminFileFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp, SolrZkClient zkClient,
+                                                 Set<String> hiddenFiles)
+      throws KeeperException, InterruptedException {
+    String adminFile = null;
+    SolrCore core = req.getCore();
+
+    final ZkSolrResourceLoader loader = (ZkSolrResourceLoader) core
+        .getResourceLoader();
+    String confPath = loader.getCollectionZkPath();
+
+    String fname = req.getParams().get("file", null);
+    if (fname == null) {
+      adminFile = confPath;
+    } else {
+      fname = fname.replace('\\', '/'); // normalize slashes
+      if (isHiddenFile(req, rsp, fname, true, hiddenFiles)) {
+        return null;
+      }
+      if (fname.startsWith("/")) { // Only files relative to conf are valid
+        fname = fname.substring(1);
+      }
+      adminFile = confPath + "/" + fname;
+    }
+
+    // Make sure the file exists, is readable and is not a hidden file
+    if (!zkClient.exists(adminFile, true)) {
+      log.error("Can not find: " + adminFile);
+      rsp.setException(new SolrException(SolrException.ErrorCode.NOT_FOUND, "Can not find: "
+          + adminFile));
+      return null;
+    }
+
+    return adminFile;
+  }
+
+
+  // Find the file indicated by the "file=XXX" parameter or the root of the conf directory on the local
+  // file system. Respects all the "interesting" stuff around what the resource loader does to find files.
+  public static File getAdminFileFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp,
+                                                Set<String> hiddenFiles) {
+    File adminFile = null;
+    final SolrResourceLoader loader = req.getCore().getResourceLoader();
+    File configdir = new File( loader.getConfigDir() );
+    if (!configdir.exists()) {
+      // TODO: maybe we should just open it this way to start with?
+      try {
+        configdir = new File( loader.getClassLoader().getResource(loader.getConfigDir()).toURI() );
+      } catch (URISyntaxException e) {
+        log.error("Can not access configuration directory!");
+        rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Can not access configuration directory!", e));
+        return null;
+      }
+    }
+    String fname = req.getParams().get("file", null);
+    if( fname == null ) {
+      adminFile = configdir;
+    }
+    else {
+      fname = fname.replace( '\\', '/' ); // normalize slashes
+      if( hiddenFiles.contains( fname.toUpperCase(Locale.ROOT) ) ) {
+        log.error("Can not access: "+ fname);
+        rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Can not access: "+fname ));
+        return null;
+      }
+      if( fname.indexOf( ".." ) >= 0 ) {
+        log.error("Invalid path: "+ fname);
+        rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Invalid path: "+fname ));
+        return null;
+      }
+      adminFile = new File( configdir, fname );
+    }
+    return adminFile;
+  }
+
+  public final Set<String> getHiddenFiles() {
+    return hiddenFiles;
+  }
+
   //////////////////////// SolrInfoMBeans methods //////////////////////
 
   @Override
   public String getDescription() {
-    return "Admin Get File -- view config files directly";
+    return "Admin Config File -- view or update config files directly";
   }
 
   @Override

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java Thu Dec 19 17:48:47 2013
@@ -48,6 +48,7 @@ public class FieldFacetStats {
   public final String name;
   final SchemaField facet_sf;
   final SchemaField field_sf;
+  final boolean calcDistinct;
 
   public final Map<String, StatsValues> facetStatsValues;
 
@@ -63,10 +64,11 @@ public class FieldFacetStats {
 
   private final BytesRef tempBR = new BytesRef();
 
-  public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf) {
+  public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf, boolean calcDistinct) {
     this.name = name;
     this.field_sf = field_sf;
     this.facet_sf = facet_sf;
+    this.calcDistinct = calcDistinct;
 
     topLevelReader = searcher.getAtomicReader();
     valueSource = facet_sf.getType().getValueSource(facet_sf, null);
@@ -78,7 +80,7 @@ public class FieldFacetStats {
   private StatsValues getStatsValues(String key) throws IOException {
     StatsValues stats = facetStatsValues.get(key);
     if (stats == null) {
-      stats = StatsValuesFactory.createStatsValues(field_sf);
+      stats = StatsValuesFactory.createStatsValues(field_sf, calcDistinct);
       facetStatsValues.put(key, stats);
       stats.setNextReader(context);
     }
@@ -139,7 +141,7 @@ public class FieldFacetStats {
       String key = (String) pairs.getKey();
       StatsValues facetStats = facetStatsValues.get(key);
       if (facetStats == null) {
-        facetStats = StatsValuesFactory.createStatsValues(field_sf);
+        facetStats = StatsValuesFactory.createStatsValues(field_sf, calcDistinct);
         facetStatsValues.put(key, facetStats);
       }
       Integer count = (Integer) pairs.getValue();

Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Thu Dec 19 17:48:47 2013
@@ -16,19 +16,6 @@ package org.apache.solr.handler.componen
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
 import org.apache.http.client.HttpClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -44,6 +31,18 @@ import org.apache.solr.util.DefaultSolrT
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
 
 public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized {
   protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
@@ -158,12 +157,7 @@ public class HttpShardHandlerFactory ext
   }
 
   protected LBHttpSolrServer createLoadbalancer(HttpClient httpClient){
-    try {
-      return new LBHttpSolrServer(httpClient);
-    } catch (MalformedURLException e) {
-      // should be impossible since we're not passing any URLs here
-      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
-    }
+    return new LBHttpSolrServer(httpClient);
   }
 
   protected <T> T getParameter(NamedList initArgs, String configKey, T defaultValue) {