You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/12/19 18:48:57 UTC
svn commit: r1552377 [9/15] - in /lucene/dev/branches/lucene5339: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/benchmark/src/ dev-tools/idea/lucene/demo/
dev-tools/idea/lucene/facet/ dev-tools/idea/solr/cont...
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/cloud/ZkController.java Thu Dec 19 17:48:47 2013
@@ -23,10 +23,10 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.BeforeReconnect;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DefaultConnectionStrategy;
import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.DocRouter;
-import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -172,11 +172,9 @@ public final class ZkController {
private int clientTimeout;
private volatile boolean isClosed;
-
- private UpdateShardHandler updateShardHandler;
public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
- String localHostContext, int leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
+ String localHostContext, int leaderVoteWait, boolean genericCoreNodeNames, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
TimeoutException, IOException {
if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null.");
this.cc = cc;
@@ -187,8 +185,6 @@ public final class ZkController {
// which means the default of "solr"
localHostContext = trimLeadingAndTrailingSlashes(localHostContext);
- updateShardHandler = new UpdateShardHandler(distribUpdateConnTimeout, distribUpdateSoTimeout);
-
this.zkServerAddress = zkServerAddress;
this.localHostPort = locaHostPort;
this.localHostContext = localHostContext;
@@ -203,46 +199,53 @@ public final class ZkController {
this.leaderVoteWait = leaderVoteWait;
this.clientTimeout = zkClientTimeout;
- zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout, zkClientConnectTimeout,
+ zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout,
+ zkClientConnectTimeout, new DefaultConnectionStrategy(),
// on reconnect, reload cloud info
new OnReconnect() {
-
+
@Override
public void command() {
try {
markAllAsNotLeader(registerOnReconnect);
- // this is troublesome - we dont want to kill anything the old leader accepted
- // though I guess sync will likely get those updates back? But only if
+ // this is troublesome - we dont want to kill anything the old
+ // leader accepted
+ // though I guess sync will likely get those updates back? But
+ // only if
// he is involved in the sync, and he certainly may not be
- // ExecutorUtil.shutdownAndAwaitTermination(cc.getCmdDistribExecutor());
+ // ExecutorUtil.shutdownAndAwaitTermination(cc.getCmdDistribExecutor());
// we need to create all of our lost watches
// seems we dont need to do this again...
- //Overseer.createClientNodes(zkClient, getNodeName());
+ // Overseer.createClientNodes(zkClient, getNodeName());
ShardHandler shardHandler;
String adminPath;
shardHandler = cc.getShardHandlerFactory().getShardHandler();
adminPath = cc.getAdminPath();
-
+
cc.cancelCoreRecoveries();
registerAllCoresAsDown(registerOnReconnect, false);
-
- ZkController.this.overseer = new Overseer(shardHandler, adminPath, zkStateReader);
- ElectionContext context = new OverseerElectionContext(zkClient, overseer, getNodeName());
+
+ ElectionContext context = new OverseerElectionContext(zkClient,
+ overseer, getNodeName());
+
overseerElector.joinElection(context, true);
zkStateReader.createClusterStateWatchersAndUpdate();
// we have to register as live first to pick up docs in the buffer
createEphemeralLiveNode();
- List<CoreDescriptor> descriptors = registerOnReconnect.getCurrentDescriptors();
+ List<CoreDescriptor> descriptors = registerOnReconnect
+ .getCurrentDescriptors();
// re register all descriptors
- if (descriptors != null) {
+ if (descriptors != null) {
for (CoreDescriptor descriptor : descriptors) {
- // TODO: we need to think carefully about what happens when it was
- // a leader that was expired - as well as what to do about leaders/overseers
+ // TODO: we need to think carefully about what happens when it
+ // was
+ // a leader that was expired - as well as what to do about
+ // leaders/overseers
// with connection loss
try {
register(descriptor.getName(), descriptor, true, true);
@@ -251,7 +254,7 @@ public final class ZkController {
}
}
}
-
+
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
@@ -262,10 +265,18 @@ public final class ZkController {
throw new ZooKeeperException(
SolrException.ErrorCode.SERVER_ERROR, "", e);
}
-
}
-
-
+
+ }, new BeforeReconnect() {
+
+ @Override
+ public void command() {
+ try {
+ ZkController.this.overseer.close();
+ } catch (Exception e) {
+ log.error("Error trying to stop any Overseer threads", e);
+ }
+ }
});
this.overseerJobQueue = Overseer.getInQueue(zkClient);
@@ -396,13 +407,6 @@ public final class ZkController {
log.error("Error closing zkClient", t);
}
- if (updateShardHandler != null) {
- try {
- updateShardHandler.close();
- } catch(Throwable t) {
- log.error("Error closing updateShardHandler", t);
- }
- }
}
/**
@@ -718,35 +722,6 @@ public final class ZkController {
return zkClient.exists(path, true);
}
- /**
- * Returns config value
- */
- public String readConfigName(String collection) throws KeeperException,
- InterruptedException {
-
- String configName = null;
-
- String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
- if (log.isInfoEnabled()) {
- log.info("Load collection config from:" + path);
- }
- byte[] data = zkClient.getData(path, null, null, true);
-
- if(data != null) {
- ZkNodeProps props = ZkNodeProps.load(data);
- configName = props.getStr(CONFIGNAME_PROP);
- }
-
- if (configName != null && !zkClient.exists(CONFIGS_ZKNODE + "/" + configName, true)) {
- log.error("Specified config does not exist in ZooKeeper:" + configName);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
- "Specified config does not exist in ZooKeeper:" + configName);
- }
-
- return configName;
- }
-
-
/**
* Register shard with ZooKeeper.
@@ -1358,30 +1333,31 @@ public final class ZkController {
public void preRegister(CoreDescriptor cd ) {
String coreNodeName = getCoreNodeName(cd);
-
- // make sure the node name is set on the descriptor
- if (cd.getCloudDescriptor().getCoreNodeName() == null) {
- cd.getCloudDescriptor().setCoreNodeName(coreNodeName);
- }
-
// before becoming available, make sure we are not live and active
// this also gets us our assigned shard id if it was not specified
try {
- if(cd.getCloudDescriptor().getCollectionName() !=null && cd.getCloudDescriptor().getCoreNodeName() != null ) {
+ CloudDescriptor cloudDesc = cd.getCloudDescriptor();
+ if(cd.getCloudDescriptor().getCollectionName() !=null && cloudDesc.getCoreNodeName() != null ) {
//we were already registered
- if(zkStateReader.getClusterState().hasCollection(cd.getCloudDescriptor().getCollectionName())){
- DocCollection coll = zkStateReader.getClusterState().getCollection(cd.getCloudDescriptor().getCollectionName());
+ if(zkStateReader.getClusterState().hasCollection(cloudDesc.getCollectionName())){
+ DocCollection coll = zkStateReader.getClusterState().getCollection(cloudDesc.getCollectionName());
if(!"true".equals(coll.getStr("autoCreated"))){
- Slice slice = coll.getSlice(cd.getCloudDescriptor().getShardId());
+ Slice slice = coll.getSlice(cloudDesc.getShardId());
if(slice != null){
- if(slice.getReplica(cd.getCloudDescriptor().getCoreNodeName()) == null) {
+ if(slice.getReplica(cloudDesc.getCoreNodeName()) == null) {
log.info("core_removed This core is removed from ZK");
- throw new SolrException(ErrorCode.NOT_FOUND,coreNodeName +" is removed");
+ throw new SolrException(ErrorCode.NOT_FOUND,cloudDesc.getCoreNodeName() +" is removed");
}
}
}
}
}
+
+ // make sure the node name is set on the descriptor
+ if (cloudDesc.getCoreNodeName() == null) {
+ cloudDesc.setCoreNodeName(coreNodeName);
+ }
+
publish(cd, ZkStateReader.DOWN, false);
} catch (KeeperException e) {
log.error("", e);
@@ -1562,11 +1538,14 @@ public final class ZkController {
return clientTimeout;
}
- // may return null if not in zk mode
- public UpdateShardHandler getUpdateShardHandler() {
- return updateShardHandler;
+ public Overseer getOverseer() {
+ return overseer;
}
+ public LeaderElector getOverseerElector() {
+ return overseerElector;
+ }
+
/**
* Returns the nodeName that should be used based on the specified properties.
*
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolr.java Thu Dec 19 17:48:47 2013
@@ -156,6 +156,14 @@ public abstract class ConfigSolr {
public int getDistributedSocketTimeout() {
return getInt(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, 0);
}
+
+ public int getMaxUpdateConnections() {
+ return getInt(CfgProp.SOLR_MAXUPDATECONNECTIONS, 10000);
+ }
+
+ public int getMaxUpdateConnectionsPerHost() {
+ return getInt(CfgProp.SOLR_MAXUPDATECONNECTIONSPERHOST, 100);
+ }
public int getCoreLoadThreadCount() {
return getInt(ConfigSolr.CfgProp.SOLR_CORELOADTHREADS, DEFAULT_CORE_LOAD_THREADS);
@@ -179,6 +187,14 @@ public abstract class ConfigSolr {
return get(CfgProp.SOLR_ADMINHANDLER, "org.apache.solr.handler.admin.CoreAdminHandler");
}
+ public String getCollectionsHandlerClass() {
+ return get(CfgProp.SOLR_COLLECTIONSHANDLER, "org.apache.solr.handler.admin.CollectionsHandler");
+ }
+
+ public String getInfoHandlerClass() {
+ return get(CfgProp.SOLR_INFOHANDLER, "org.apache.solr.handler.admin.InfoHandler");
+ }
+
public boolean hasSchemaCache() {
return getBool(ConfigSolr.CfgProp.SOLR_SHARESCHEMA, false);
}
@@ -203,13 +219,17 @@ public abstract class ConfigSolr {
// Ugly for now, but we'll at least be able to centralize all of the differences between 4x and 5x.
protected static enum CfgProp {
SOLR_ADMINHANDLER,
+ SOLR_COLLECTIONSHANDLER,
SOLR_CORELOADTHREADS,
SOLR_COREROOTDIRECTORY,
SOLR_DISTRIBUPDATECONNTIMEOUT,
SOLR_DISTRIBUPDATESOTIMEOUT,
+ SOLR_MAXUPDATECONNECTIONS,
+ SOLR_MAXUPDATECONNECTIONSPERHOST,
SOLR_HOST,
SOLR_HOSTCONTEXT,
SOLR_HOSTPORT,
+ SOLR_INFOHANDLER,
SOLR_LEADERVOTEWAIT,
SOLR_LOGGING_CLASS,
SOLR_LOGGING_ENABLED,
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java Thu Dec 19 17:48:47 2013
@@ -103,10 +103,14 @@ public class ConfigSolrXml extends Confi
private void fillPropMap() {
propMap.put(CfgProp.SOLR_ADMINHANDLER, doSub("solr/str[@name='adminHandler']"));
+ propMap.put(CfgProp.SOLR_COLLECTIONSHANDLER, doSub("solr/str[@name='collectionsHandler']"));
+ propMap.put(CfgProp.SOLR_INFOHANDLER, doSub("solr/str[@name='infoHandler']"));
propMap.put(CfgProp.SOLR_CORELOADTHREADS, doSub("solr/int[@name='coreLoadThreads']"));
propMap.put(CfgProp.SOLR_COREROOTDIRECTORY, doSub("solr/str[@name='coreRootDirectory']"));
propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateConnTimeout']"));
propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, doSub("solr/solrcloud/int[@name='distribUpdateSoTimeout']"));
+ propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONS, doSub("solr/solrcloud/int[@name='maxUpdateConnections']"));
+ propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONSPERHOST, doSub("solr/solrcloud/int[@name='maxUpdateConnectionsPerHost']"));
propMap.put(CfgProp.SOLR_HOST, doSub("solr/solrcloud/str[@name='host']"));
propMap.put(CfgProp.SOLR_HOSTCONTEXT, doSub("solr/solrcloud/str[@name='hostContext']"));
propMap.put(CfgProp.SOLR_HOSTPORT, doSub("solr/solrcloud/int[@name='hostPort']"));
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Thu Dec 19 17:48:47 2013
@@ -141,10 +141,16 @@ public class ConfigSolrXmlOld extends Co
propMap.put(CfgProp.SOLR_ADMINHANDLER,
config.getVal("solr/cores/@adminHandler", false));
+ propMap.put(CfgProp.SOLR_COLLECTIONSHANDLER, config.getVal("solr/cores/@collectionsHandler", false));
+ propMap.put(CfgProp.SOLR_INFOHANDLER, config.getVal("solr/cores/@infoHandler", false));
propMap.put(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT,
config.getVal("solr/cores/@distribUpdateConnTimeout", false));
propMap.put(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT,
config.getVal("solr/cores/@distribUpdateSoTimeout", false));
+ propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONS,
+ config.getVal("solr/cores/@maxUpdateConnections", false));
+ propMap.put(CfgProp.SOLR_MAXUPDATECONNECTIONSPERHOST,
+ config.getVal("solr/cores/@maxUpdateConnectionsPerHost", false));
propMap.put(CfgProp.SOLR_HOST, config.getVal("solr/cores/@host", false));
propMap.put(CfgProp.SOLR_HOSTCONTEXT,
config.getVal("solr/cores/@hostContext", false));
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoreContainer.java Thu Dec 19 17:48:47 2013
@@ -17,7 +17,26 @@
package org.apache.solr.core;
-import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.collect.Maps;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.handler.admin.CoreAdminHandler;
+import org.apache.solr.handler.admin.InfoHandler;
+import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.logging.LogWatcher;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.IndexSchemaFactory;
+import org.apache.solr.update.UpdateShardHandler;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.FileUtils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.text.SimpleDateFormat;
@@ -40,27 +59,7 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.cloud.ZkSolrResourceLoader;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.ZooKeeperException;
-import org.apache.solr.common.util.ExecutorUtil;
-import org.apache.solr.common.util.SolrjNamedThreadFactory;
-import org.apache.solr.handler.admin.CollectionsHandler;
-import org.apache.solr.handler.admin.CoreAdminHandler;
-import org.apache.solr.handler.admin.InfoHandler;
-import org.apache.solr.handler.component.ShardHandlerFactory;
-import org.apache.solr.logging.LogWatcher;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.IndexSchemaFactory;
-import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.util.FileUtils;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
+import static com.google.common.base.Preconditions.checkNotNull;
/**
@@ -88,8 +87,7 @@ public class CoreContainer {
protected ZkContainer zkSys = new ZkContainer();
private ShardHandlerFactory shardHandlerFactory;
- private ExecutorService updateExecutor = Executors.newCachedThreadPool(
- new SolrjNamedThreadFactory("updateExecutor"));
+ private UpdateShardHandler updateShardHandler;
protected LogWatcher logging = null;
@@ -102,6 +100,8 @@ public class CoreContainer {
protected final CoresLocator coresLocator;
private String hostName;
+
+ // private ClientConnectionManager clientConnectionManager = new PoolingClientConnectionManager();
{
log.info("New CoreContainer " + System.identityHashCode(this));
@@ -196,6 +196,8 @@ public class CoreContainer {
}
shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
+
+ updateShardHandler = new UpdateShardHandler(cfg);
solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
@@ -212,9 +214,9 @@ public class CoreContainer {
zkSys.initZooKeeper(this, solrHome, cfg);
- collectionsHandler = new CollectionsHandler(this);
- infoHandler = new InfoHandler(this);
- coreAdminHandler = createMultiCoreHandler(cfg.getCoreAdminHandlerClass());
+ collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
+ infoHandler = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class);
+ coreAdminHandler = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
containerProperties = cfg.getSolrProperties("solr");
@@ -358,7 +360,6 @@ public class CoreContainer {
cancelCoreRecoveries();
}
-
try {
// First wake up the closer thread, it'll terminate almost immediately since it checks isShutDown.
synchronized (solrCores.getModifyLock()) {
@@ -384,16 +385,20 @@ public class CoreContainer {
}
} finally {
- if (shardHandlerFactory != null) {
- shardHandlerFactory.close();
+ try {
+ if (shardHandlerFactory != null) {
+ shardHandlerFactory.close();
+ }
+ } finally {
+ try {
+ if (updateShardHandler != null) {
+ updateShardHandler.close();
+ }
+ } finally {
+ // we want to close zk stuff last
+ zkSys.close();
+ }
}
-
- ExecutorUtil.shutdownAndAwaitTermination(updateExecutor);
-
- // we want to close zk stuff last
-
- zkSys.close();
-
}
org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort
}
@@ -676,7 +681,7 @@ public class CoreContainer {
String collection = cd.getCloudDescriptor().getCollectionName();
zkSys.getZkController().createCollectionZkNode(cd.getCloudDescriptor());
- String zkConfigName = zkSys.getZkController().readConfigName(collection);
+ String zkConfigName = zkSys.getZkController().getZkStateReader().readConfigName(collection);
if (zkConfigName == null) {
log.error("Could not find config name for collection:" + collection);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
@@ -726,7 +731,7 @@ public class CoreContainer {
n1 = checkDefault(n1);
solrCores.swap(n0, n1);
- coresLocator.persist(this, solrCores.getCoreDescriptor(n0), solrCores.getCoreDescriptor(n1));
+ coresLocator.swap(this, solrCores.getCoreDescriptor(n0), solrCores.getCoreDescriptor(n1));
log.info("swapped: "+n0 + " with " + n1);
}
@@ -773,10 +778,10 @@ public class CoreContainer {
return null;
}
- /**
+ /**
* Gets a core by name and increase its refcount.
*
- * @see SolrCore#close()
+ * @see SolrCore#close()
* @param name the core name
* @return the core if found, null if a SolrCore by this name does not exist
* @exception SolrException if a SolrCore with this name failed to be initialized
@@ -795,7 +800,7 @@ public class CoreContainer {
// OK, it's not presently in any list, is it in the list of dynamic cores but not loaded yet? If so, load it.
CoreDescriptor desc = solrCores.getDynamicDescriptor(name);
if (desc == null) { //Nope, no transient core with this name
-
+
// if there was an error initalizing this core, throw a 500
// error with the details for clients attempting to access it.
Exception e = getCoreInitFailures().get(name);
@@ -826,7 +831,7 @@ public class CoreContainer {
}
} catch(Exception ex){
// remains to be seen how transient cores and such
- // will work in SolrCloud mode, but just to be future
+ // will work in SolrCloud mode, but just to be future
// proof...
/*if (isZooKeeperAware()) {
try {
@@ -846,34 +851,33 @@ public class CoreContainer {
return core;
}
- // ---------------- Multicore self related methods ---------------
- /**
- * Creates a CoreAdminHandler for this MultiCore.
- * @return a CoreAdminHandler
- */
- protected CoreAdminHandler createMultiCoreHandler(final String adminHandlerClass) {
- return loader.newAdminHandlerInstance(CoreContainer.this, adminHandlerClass);
+ // ---------------- CoreContainer request handlers --------------
+
+ protected <T> T createHandler(String handlerClass, Class<T> clazz) {
+ return loader.newInstance(handlerClass, clazz, null, new Class[] { CoreContainer.class }, new Object[] { this });
}
public CoreAdminHandler getMultiCoreHandler() {
return coreAdminHandler;
}
-
+
public CollectionsHandler getCollectionsHandler() {
return collectionsHandler;
}
-
+
public InfoHandler getInfoHandler() {
return infoHandler;
}
-
+
+ // ---------------- Multicore self related methods ---------------
+
/**
* the default core name, or null if there is no default core name
*/
public String getDefaultCoreName() {
return cfg.getDefaultCoreName();
}
-
+
// all of the following properties aren't synchronized
// but this should be OK since they normally won't be changed rapidly
@Deprecated
@@ -952,8 +956,8 @@ public class CoreContainer {
return shardHandlerFactory;
}
- public ExecutorService getUpdateExecutor() {
- return updateExecutor;
+ public UpdateShardHandler getUpdateShardHandler() {
+ return updateShardHandler;
}
// Just to tidy up the code where it did this in-line.
@@ -969,8 +973,6 @@ public class CoreContainer {
String getCoreToOrigName(SolrCore core) {
return solrCores.getCoreToOrigName(core);
}
-
-
}
class CloserThread extends Thread {
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java Thu Dec 19 17:48:47 2013
@@ -91,7 +91,11 @@ public class CorePropertiesLocator imple
@Override
public void delete(CoreContainer cc, CoreDescriptor... coreDescriptors) {
+ if (coreDescriptors == null) {
+ return;
+ }
for (CoreDescriptor cd : coreDescriptors) {
+ if (cd == null) continue;
File instanceDir = new File(cd.getInstanceDir());
File propertiesFile = new File(instanceDir, PROPERTIES_FILENAME);
propertiesFile.renameTo(new File(instanceDir, PROPERTIES_FILENAME + ".unloaded"));
@@ -107,6 +111,11 @@ public class CorePropertiesLocator imple
}
@Override
+ public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2) {
+ persist(cc, cd1, cd2);
+ }
+
+ @Override
public List<CoreDescriptor> discover(CoreContainer cc) {
logger.info("Looking for core definitions underneath {}", rootDirectory.getAbsolutePath());
List<CoreDescriptor> cds = Lists.newArrayList();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/CoresLocator.java Thu Dec 19 17:48:47 2013
@@ -56,6 +56,14 @@ public interface CoresLocator {
public void rename(CoreContainer cc, CoreDescriptor oldCD, CoreDescriptor newCD);
/**
+ * Swap two core definitions
+ * @param cc the CoreContainer
+ * @param cd1 the core descriptor of the first core, after swapping
+ * @param cd2 the core descriptor of the second core, after swapping
+ */
+ public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2);
+
+ /**
* Load all the CoreDescriptors from persistence store
* @param cc the CoreContainer
* @return a list of all CoreDescriptors found
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java Thu Dec 19 17:48:47 2013
@@ -320,7 +320,7 @@ public class JmxMonitoredMap<K, V> exten
try {
list.add(new Attribute(attribute, getAttribute(attribute)));
} catch (Exception e) {
- LOG.warn("Could not get attibute " + attribute);
+ LOG.warn("Could not get attribute " + attribute);
}
}
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrCore.java Thu Dec 19 17:48:47 2013
@@ -17,9 +17,42 @@
package org.apache.solr.core;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.xml.parsers.ParserConfigurationException;
+
import org.apache.commons.io.IOUtils;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriter;
@@ -40,6 +73,7 @@ import org.apache.solr.common.util.Simpl
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.handler.SnapPuller;
import org.apache.solr.handler.admin.ShowFileRequestHandler;
+import org.apache.solr.handler.component.AnalyticsComponent;
import org.apache.solr.handler.component.DebugComponent;
import org.apache.solr.handler.component.FacetComponent;
import org.apache.solr.handler.component.HighlightComponent;
@@ -93,39 +127,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Writer;
-import java.lang.reflect.Constructor;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.StringTokenizer;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-
/**
*
@@ -769,7 +770,7 @@ public final class SolrCore implements S
updateProcessorChains = loadUpdateProcessorChains();
reqHandlers = new RequestHandlers(this);
reqHandlers.initHandlersFromConfig(solrConfig);
-
+
// Handle things that should eventually go away
initDeprecatedSupport();
@@ -853,13 +854,21 @@ public final class SolrCore implements S
CoreContainer cc = cd.getCoreContainer();
- if (cc != null && cc.isZooKeeperAware() && Slice.CONSTRUCTION.equals(cd.getCloudDescriptor().getShardState())) {
- // set update log to buffer before publishing the core
- getUpdateHandler().getUpdateLog().bufferUpdates();
-
- cd.getCloudDescriptor().setShardState(null);
- cd.getCloudDescriptor().setShardRange(null);
- cd.getCloudDescriptor().setShardParent(null);
+ if (cc != null && cc.isZooKeeperAware()) {
+ SolrRequestHandler realtimeGetHandler = reqHandlers.get("/get");
+ if (realtimeGetHandler == null) {
+ log.warn("WARNING: RealTimeGetHandler is not registered at /get. " +
+ "SolrCloud will always use full index replication instead of the more efficient PeerSync method.");
+ }
+
+ if (Slice.CONSTRUCTION.equals(cd.getCloudDescriptor().getShardState())) {
+ // set update log to buffer before publishing the core
+ getUpdateHandler().getUpdateLog().bufferUpdates();
+
+ cd.getCloudDescriptor().setShardState(null);
+ cd.getCloudDescriptor().setShardRange(null);
+ cd.getCloudDescriptor().setShardParent(null);
+ }
}
// For debugging
// numOpens.incrementAndGet();
@@ -1197,6 +1206,7 @@ public final class SolrCore implements S
addIfNotPresent(components,StatsComponent.COMPONENT_NAME,StatsComponent.class);
addIfNotPresent(components,DebugComponent.COMPONENT_NAME,DebugComponent.class);
addIfNotPresent(components,RealTimeGetComponent.COMPONENT_NAME,RealTimeGetComponent.class);
+ addIfNotPresent(components,AnalyticsComponent.COMPONENT_NAME,AnalyticsComponent.class);
return components;
}
private <T> void addIfNotPresent(Map<String ,T> registry, String name, Class<? extends T> c){
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Thu Dec 19 17:48:47 2013
@@ -79,7 +79,7 @@ public class SolrResourceLoader implemen
static final String project = "solr";
static final String base = "org.apache" + "." + project;
- static final String[] packages = {"","analysis.","schema.","handler.","search.","update.","core.","response.","request.","update.processor.","util.", "spelling.", "handler.component.", "handler.dataimport." };
+ static final String[] packages = {"","analysis.","schema.","handler.","search.","update.","core.","response.","request.","update.processor.","util.", "spelling.", "handler.component.", "handler.dataimport.", "spelling.suggest.", "spelling.suggest.fst." };
protected URLClassLoader classLoader;
private final String instanceDir;
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java Thu Dec 19 17:48:47 2013
@@ -19,7 +19,6 @@ package org.apache.solr.core;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -32,11 +31,9 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -191,6 +188,11 @@ public class SolrXMLCoresLocator impleme
}
@Override
+ public void swap(CoreContainer cc, CoreDescriptor cd1, CoreDescriptor cd2) {
+ this.persist(cc);
+ }
+
+ @Override
public List<CoreDescriptor> discover(CoreContainer cc) {
ImmutableList.Builder<CoreDescriptor> listBuilder = ImmutableList.builder();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/core/ZkContainer.java Thu Dec 19 17:48:47 2013
@@ -51,8 +51,6 @@ public class ZkContainer {
private String host;
private int leaderVoteWait;
private Boolean genericCoreNodeNames;
- private int distribUpdateConnTimeout;
- private int distribUpdateSoTimeout;
public ZkContainer() {
@@ -67,13 +65,11 @@ public class ZkContainer {
initZooKeeper(cc, solrHome,
config.getZkHost(), config.getZkClientTimeout(), config.getZkHostPort(), config.getZkHostContext(),
- config.getHost(), config.getLeaderVoteWait(), config.getGenericCoreNodeNames(),
- config.getDistributedConnectionTimeout(), config.getDistributedSocketTimeout());
+ config.getHost(), config.getLeaderVoteWait(), config.getGenericCoreNodeNames());
}
public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort,
- String hostContext, String host, int leaderVoteWait, boolean genericCoreNodeNames,
- int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
+ String hostContext, String host, int leaderVoteWait, boolean genericCoreNodeNames) {
ZkController zkController = null;
// if zkHost sys property is not set, we are not using ZooKeeper
@@ -92,8 +88,6 @@ public class ZkContainer {
this.host = host;
this.leaderVoteWait = leaderVoteWait;
this.genericCoreNodeNames = genericCoreNodeNames;
- this.distribUpdateConnTimeout = distribUpdateConnTimeout;
- this.distribUpdateSoTimeout = distribUpdateSoTimeout;
if (zkRun == null && zookeeperHost == null)
return; // not in zk mode
@@ -125,7 +119,7 @@ public class ZkContainer {
}
}
- int zkClientConnectTimeout = 15000;
+ int zkClientConnectTimeout = 30000;
if (zookeeperHost != null) {
@@ -147,7 +141,7 @@ public class ZkContainer {
}
zkController = new ZkController(cc, zookeeperHost, zkClientTimeout,
zkClientConnectTimeout, host, hostPort, hostContext,
- leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout,
+ leaderVoteWait, genericCoreNodeNames,
new CurrentCoreDescriptorProvider() {
@Override
@@ -220,7 +214,7 @@ public class ZkContainer {
String collection = dcore.getCloudDescriptor().getCollectionName();
zkController.createCollectionZkNode(dcore.getCloudDescriptor());
- zkConfigName = zkController.readConfigName(collection);
+ zkConfigName = zkController.getZkStateReader().readConfigName(collection);
if (zkConfigName == null) {
log.error("Could not find config name for collection:" + collection);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Thu Dec 19 17:48:47 2013
@@ -16,6 +16,7 @@
*/
package org.apache.solr.handler;
+import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
import static org.apache.solr.handler.ReplicationHandler.ALIAS;
import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
@@ -47,6 +48,7 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@@ -55,6 +57,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -75,9 +78,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-
-import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
-
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -104,6 +104,7 @@ import org.apache.solr.util.FileUtils;
import org.apache.solr.util.PropertiesInputStream;
import org.apache.solr.util.PropertiesOutputStream;
import org.apache.solr.util.RefCounted;
+import org.eclipse.jetty.util.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -162,25 +163,18 @@ public class SnapPuller {
*/
private AtomicBoolean pollDisabled = new AtomicBoolean(false);
- // HttpClient shared by all cores (used if timeout is not specified for a core)
- private static HttpClient client;
- // HttpClient for this instance if connectionTimeout or readTimeout has been specified
private final HttpClient myHttpClient;
- private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) {
- if (connTimeout == null && readTimeout == null && client != null) return client;
+ private static HttpClient createHttpClient(SolrCore core, String connTimeout, String readTimeout, String httpBasicAuthUser, String httpBasicAuthPassword, boolean useCompression) {
final ModifiableSolrParams httpClientParams = new ModifiableSolrParams();
httpClientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, connTimeout != null ? connTimeout : "5000");
httpClientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, readTimeout != null ? readTimeout : "20000");
httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser);
httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword);
httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression);
- // Keeping a very high number so that if you have a large number of cores
- // no requests are kept waiting for an idle connection.
- httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000);
- httpClientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 10000);
- HttpClient httpClient = HttpClientUtil.createClient(httpClientParams);
- if (client == null && connTimeout == null && readTimeout == null) client = httpClient;
+
+ HttpClient httpClient = HttpClientUtil.createClient(httpClientParams, core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getConnectionManager());
+
return httpClient;
}
@@ -207,7 +201,7 @@ public class SnapPuller {
String readTimeout = (String) initArgs.get(HttpClientUtil.PROP_SO_TIMEOUT);
String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
- myHttpClient = createHttpClient(connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, useExternal);
+ myHttpClient = createHttpClient(solrCore, connTimeout, readTimeout, httpBasicAuthUser, httpBasicAuthPassword, useExternal);
if (pollInterval != null && pollInterval > 0) {
startExecutorService();
} else {
@@ -388,8 +382,8 @@ public class SnapPuller {
fsyncService = Executors.newSingleThreadExecutor(new DefaultSolrThreadFactory("fsyncService"));
// use a synchronized list because the list is read by other threads (to show details)
filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
- // if the generateion of master is older than that of the slave , it means they are not compatible to be copied
- // then a new index direcory to be created and all the files need to be copied
+ // if the generation of master is older than that of the slave , it means they are not compatible to be copied
+ // then a new index directory to be created and all the files need to be copied
boolean isFullCopyNeeded = IndexDeletionPolicyWrapper
.getCommitTimestamp(commit) >= latestVersion
|| commit.getGeneration() >= latestGeneration || forceReplication;
@@ -408,59 +402,76 @@ public class SnapPuller {
if (isIndexStale(indexDir)) {
isFullCopyNeeded = true;
}
- LOG.info("Starting download to " + tmpIndexDir + " fullCopy=" + isFullCopyNeeded);
- successfulInstall = false;
- downloadIndexFiles(isFullCopyNeeded, tmpIndexDir, latestGeneration);
- LOG.info("Total time taken for download : " + ((System.currentTimeMillis() - replicationStartTime) / 1000) + " secs");
- Collection<Map<String, Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
- if (!modifiedConfFiles.isEmpty()) {
- downloadConfFiles(confFilesToDownload, latestGeneration);
- if (isFullCopyNeeded) {
- successfulInstall = modifyIndexProps(tmpIdxDirName);
- deleteTmpIdxDir = false;
- } else {
- solrCore.getUpdateHandler().getSolrCoreState()
- .closeIndexWriter(core, true);
- try {
+ if (!isFullCopyNeeded) {
+ // rollback - and do it before we download any files
+ // so we don't remove files we thought we didn't need
+ // to download later
+ solrCore.getUpdateHandler().getSolrCoreState()
+ .closeIndexWriter(core, true);
+ }
+
+ boolean reloadCore = false;
+
+ try {
+ LOG.info("Starting download to " + tmpIndexDir + " fullCopy="
+ + isFullCopyNeeded);
+ successfulInstall = false;
+
+ downloadIndexFiles(isFullCopyNeeded, indexDir, tmpIndexDir,
+ latestGeneration);
+ LOG.info("Total time taken for download : "
+ + ((System.currentTimeMillis() - replicationStartTime) / 1000)
+ + " secs");
+ Collection<Map<String,Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
+ if (!modifiedConfFiles.isEmpty()) {
+ downloadConfFiles(confFilesToDownload, latestGeneration);
+ if (isFullCopyNeeded) {
+ successfulInstall = modifyIndexProps(tmpIdxDirName);
+ deleteTmpIdxDir = false;
+ } else {
successfulInstall = moveIndexFiles(tmpIndexDir, indexDir);
- } finally {
- solrCore.getUpdateHandler().getSolrCoreState()
- .openIndexWriter(core);
}
- }
- if (successfulInstall) {
- if (isFullCopyNeeded) {
- // let the system know we are changing dir's and the old one
- // may be closed
- if (indexDir != null) {
- LOG.info("removing old index directory " + indexDir);
- core.getDirectoryFactory().doneWithDirectory(indexDir);
- core.getDirectoryFactory().remove(indexDir);
+ if (successfulInstall) {
+ if (isFullCopyNeeded) {
+ // let the system know we are changing dir's and the old one
+ // may be closed
+ if (indexDir != null) {
+ LOG.info("removing old index directory " + indexDir);
+ core.getDirectoryFactory().doneWithDirectory(indexDir);
+ core.getDirectoryFactory().remove(indexDir);
+ }
}
+
+ LOG.info("Configuration files are modified, core will be reloaded");
+ logReplicationTimeAndConfFiles(modifiedConfFiles,
+ successfulInstall);// write to a file time of replication and
+ // conf files.
+ reloadCore = true;
}
-
- LOG.info("Configuration files are modified, core will be reloaded");
- logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);//write to a file time of replication and conf files.
- reloadCore();
- }
- } else {
- terminateAndWaitFsyncService();
- if (isFullCopyNeeded) {
- successfulInstall = modifyIndexProps(tmpIdxDirName);
- deleteTmpIdxDir = false;
} else {
- solrCore.getUpdateHandler().getSolrCoreState().closeIndexWriter(core, true);
- try {
+ terminateAndWaitFsyncService();
+ if (isFullCopyNeeded) {
+ successfulInstall = modifyIndexProps(tmpIdxDirName);
+ deleteTmpIdxDir = false;
+ } else {
successfulInstall = moveIndexFiles(tmpIndexDir, indexDir);
- } finally {
- solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(core);
+ }
+ if (successfulInstall) {
+ logReplicationTimeAndConfFiles(modifiedConfFiles,
+ successfulInstall);
}
}
- if (successfulInstall) {
- logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);
+ } finally {
+ if (!isFullCopyNeeded) {
+ solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(core);
}
}
+
+ // we must reload the core after we open the IW back up
+ if (reloadCore) {
+ reloadCore();
+ }
if (successfulInstall) {
if (isFullCopyNeeded) {
@@ -690,6 +701,7 @@ public class SnapPuller {
}
private void reloadCore() {
+ final CountDownLatch latch = new CountDownLatch(1);
new Thread() {
@Override
public void run() {
@@ -697,9 +709,17 @@ public class SnapPuller {
solrCore.getCoreDescriptor().getCoreContainer().reload(solrCore.getName());
} catch (Exception e) {
LOG.error("Could not reload core ", e);
+ } finally {
+ latch.countDown();
}
}
}.start();
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while waiting for core reload to finish", e);
+ }
}
private void downloadConfFiles(List<Map<String, Object>> confFilesToDownload, long latestGeneration) throws Exception {
@@ -732,29 +752,28 @@ public class SnapPuller {
* Download the index files. If a new index is needed, download all the files.
*
* @param downloadCompleteIndex is it a fresh index copy
- * @param tmpIndexDir the directory to which files need to be downloadeed to
+ * @param tmpIndexDir the directory to which files need to be downloadeed to
+ * @param indexDir the indexDir to be merged to
* @param latestGeneration the version number
*/
private void downloadIndexFiles(boolean downloadCompleteIndex,
- Directory tmpIndexDir, long latestGeneration) throws Exception {
- String indexDir = solrCore.getIndexDir();
-
- // it's okay to use null for lock factory since we know this dir will exist
- Directory dir = solrCore.getDirectoryFactory().get(indexDir, DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
- try {
- for (Map<String,Object> file : filesToDownload) {
- if (!dir.fileExists((String) file.get(NAME)) || downloadCompleteIndex) {
- dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file,
- (String) file.get(NAME), false, latestGeneration);
- currentFile = file;
- dirFileFetcher.fetchFile();
- filesDownloaded.add(new HashMap<String,Object>(file));
- } else {
- LOG.info("Skipping download for " + file.get(NAME) + " because it already exists");
- }
+ Directory indexDir, Directory tmpIndexDir, long latestGeneration)
+ throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Download files to dir: " + Arrays.asList(indexDir.listAll()));
+ }
+ for (Map<String,Object> file : filesToDownload) {
+ if (!indexDir.fileExists((String) file.get(NAME))
+ || downloadCompleteIndex) {
+ dirFileFetcher = new DirectoryFileFetcher(tmpIndexDir, file,
+ (String) file.get(NAME), false, latestGeneration);
+ currentFile = file;
+ dirFileFetcher.fetchFile();
+ filesDownloaded.add(new HashMap<String,Object>(file));
+ } else {
+ LOG.info("Skipping download for " + file.get(NAME)
+ + " because it already exists");
}
- } finally {
- solrCore.getDirectoryFactory().release(dir);
}
}
@@ -769,6 +788,8 @@ public class SnapPuller {
for (Map<String, Object> file : filesToDownload) {
if (dir.fileExists((String) file.get(NAME))
&& dir.fileLength((String) file.get(NAME)) != (Long) file.get(SIZE)) {
+ LOG.warn("File " + file.get(NAME) + " expected to be " + file.get(SIZE)
+ + " while it is " + dir.fileLength((String) file.get(NAME)));
// file exists and size is different, therefore we must assume
// corrupted index
return true;
@@ -782,6 +803,7 @@ public class SnapPuller {
* <p/>
*/
private boolean moveAFile(Directory tmpIdxDir, Directory indexDir, String fname, List<String> copiedfiles) {
+ LOG.debug("Moving file: {}", fname);
boolean success = false;
try {
if (indexDir.fileExists(fname)) {
@@ -805,6 +827,14 @@ public class SnapPuller {
* Copy all index files from the temp index dir to the actual index. The segments_N file is copied last.
*/
private boolean moveIndexFiles(Directory tmpIdxDir, Directory indexDir) {
+ if (LOG.isDebugEnabled()) {
+ try {
+ LOG.info("From dir files:" + Arrays.asList(tmpIdxDir.listAll()));
+ LOG.info("To dir files:" + Arrays.asList(indexDir.listAll()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
String segmentsFile = null;
List<String> movedfiles = new ArrayList<String>();
for (Map<String, Object> f : filesDownloaded) {
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java Thu Dec 19 17:48:47 2013
@@ -75,7 +75,7 @@ public class UpdateRequestHandler extend
type = stream.getContentType();
}
if( type == null ) { // Normal requests will not get here.
- throw new SolrException(ErrorCode.BAD_REQUEST, "Missing ContentType");
+ throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Missing ContentType");
}
int idx = type.indexOf(';');
if(idx>0) {
@@ -83,7 +83,7 @@ public class UpdateRequestHandler extend
}
ContentStreamLoader loader = loaders.get(type);
if(loader==null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported ContentType: "
+ throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Unsupported ContentType: "
+type+ " Not in: "+loaders.keySet());
}
if(loader.getDefaultWT()!=null) {
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Thu Dec 19 17:48:47 2013
@@ -318,6 +318,7 @@ public class CollectionsHandler extends
SHARDS_PROP,
"router.");
+ copyPropertiesIfNotNull(req.getParams(), props);
ZkNodeProps m = new ZkNodeProps(props);
handleResponse(OverseerCollectionProcessor.CREATECOLLECTION, m, rsp);
@@ -342,6 +343,7 @@ public class CollectionsHandler extends
Map<String, Object> map = makeMap(QUEUE_OPERATION, CREATESHARD);
copyIfNotNull(req.getParams(),map,COLLECTION_PROP, SHARD_ID_PROP, REPLICATION_FACTOR,CREATE_NODE_SET);
+ copyPropertiesIfNotNull(req.getParams(), map);
ZkNodeProps m = new ZkNodeProps(map);
handleResponse(CREATESHARD, m, rsp);
}
@@ -372,7 +374,18 @@ public class CollectionsHandler extends
}
}
-
+
+ private void copyPropertiesIfNotNull(SolrParams params, Map<String, Object> props) {
+ Iterator<String> iter = params.getParameterNamesIterator();
+ while (iter.hasNext()) {
+ String param = iter.next();
+ if (param.startsWith(OverseerCollectionProcessor.COLL_PROP_PREFIX)) {
+ props.put(param, params.get(param));
+ }
+ }
+ }
+
+
private void handleDeleteShardAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws InterruptedException, KeeperException {
log.info("Deleting Shard : " + req.getParamString());
@@ -420,6 +433,7 @@ public class CollectionsHandler extends
if (rangesStr != null) {
props.put(CoreAdminParams.RANGES, rangesStr);
}
+ copyPropertiesIfNotNull(req.getParams(), props);
ZkNodeProps m = new ZkNodeProps(props);
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Thu Dec 19 17:48:47 2013
@@ -596,6 +596,15 @@ public class CoreAdminHandler extends Re
"No such core exists '" + cname + "'");
} else {
if (coreContainer.getZkController() != null) {
+ // we are unloading, cancel any ongoing recovery
+ // so there are no races to publish state
+ // we will try to cancel again later before close
+ if (core != null) {
+ if (coreContainer.getZkController() != null) {
+ core.getSolrCoreState().cancelRecovery();
+ }
+ }
+
log.info("Unregistering core " + core.getName() + " from cloudstate.");
try {
coreContainer.getZkController().unregister(cname,
@@ -796,7 +805,7 @@ public class CoreAdminHandler extends Re
try {
core = coreContainer.getCore(cname);
if (core != null) {
- syncStrategy = new SyncStrategy();
+ syncStrategy = new SyncStrategy(core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler());
Map<String,Object> props = new HashMap<String,Object>();
props.put(ZkStateReader.BASE_URL_PROP, zkController.getBaseUrl());
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java Thu Dec 19 17:48:47 2013
@@ -23,6 +23,7 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -33,7 +34,11 @@ import org.apache.solr.handler.RequestHa
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.ManagedIndexSchema;
import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -60,10 +65,15 @@ import java.util.Set;
* </lst>
* <lst name="invariants">
* <str name="hidden">synonyms.txt</str>
- * <str name="hidden">anotherfile.txt</str>
+ * <str name="hidden">anotherfile.txt</str>
+ * <str name="hidden">*</str>
* </lst>
* </requestHandler>
* </pre>
+ *
+ * At present, there is only explicit file names (including path) or the glob '*' are supported. Variants like '*.xml'
+ * are NOT supported.ere
+ *
* <p>
* The ShowFileRequestHandler uses the {@link RawResponseWriter} (wt=raw) to return
* file contents. If you need to use a different writer, you will need to change
@@ -75,7 +85,7 @@ import java.util.Set;
* <pre>
* http://localhost:8983/solr/admin/file?file=schema.xml&contentType=text/plain
* </pre>
- *
+ *
*
* @since solr 1.3
*/
@@ -83,9 +93,13 @@ public class ShowFileRequestHandler exte
{
public static final String HIDDEN = "hidden";
public static final String USE_CONTENT_TYPE = "contentType";
-
+
protected Set<String> hiddenFiles;
-
+
+ protected static final Logger log = LoggerFactory
+ .getLogger(ShowFileRequestHandler.class);
+
+
public ShowFileRequestHandler()
{
super();
@@ -94,27 +108,28 @@ public class ShowFileRequestHandler exte
@Override
public void init(NamedList args) {
super.init( args );
+ hiddenFiles = initHidden(invariants);
+ }
+
+ public static Set<String> initHidden(SolrParams invariants) {
+ Set<String> hiddenRet = new HashSet<String>();
// Build a list of hidden files
- hiddenFiles = new HashSet<String>();
- if( invariants != null ) {
- String[] hidden = invariants.getParams( HIDDEN );
- if( hidden != null ) {
- for( String s : hidden ) {
- hiddenFiles.add( s.toUpperCase(Locale.ROOT) );
+ if (invariants != null) {
+ String[] hidden = invariants.getParams(HIDDEN);
+ if (hidden != null) {
+ for (String s : hidden) {
+ hiddenRet.add(s.toUpperCase(Locale.ROOT));
}
}
}
+ return hiddenRet;
}
-
- public Set<String> getHiddenFiles()
- {
- return hiddenFiles;
- }
-
+
@Override
- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, KeeperException, InterruptedException
- {
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
+ throws InterruptedException, KeeperException, IOException {
+
CoreContainer coreContainer = req.getCore().getCoreDescriptor().getCoreContainer();
if (coreContainer.isZooKeeperAware()) {
showFromZooKeeper(req, rsp, coreContainer);
@@ -123,58 +138,32 @@ public class ShowFileRequestHandler exte
}
}
+ // Get a list of files from ZooKeeper for from the path in the file= parameter.
private void showFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp,
CoreContainer coreContainer) throws KeeperException,
InterruptedException, UnsupportedEncodingException {
- String adminFile = null;
- SolrCore core = req.getCore();
+
SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
- final ZkSolrResourceLoader loader = (ZkSolrResourceLoader) core
- .getResourceLoader();
- String confPath = loader.getCollectionZkPath();
-
- String fname = req.getParams().get("file", null);
- if (fname == null) {
- adminFile = confPath;
- } else {
- fname = fname.replace('\\', '/'); // normalize slashes
- if (hiddenFiles.contains(fname.toUpperCase(Locale.ROOT))) {
- rsp.setException(new SolrException(ErrorCode.FORBIDDEN, "Can not access: " + fname));
- return;
- }
- if (fname.indexOf("..") >= 0) {
- rsp.setException(new SolrException(ErrorCode.FORBIDDEN, "Invalid path: " + fname));
- return;
- }
- if (fname.startsWith("/")) { // Only files relative to conf are valid
- fname = fname.substring(1);
- }
- adminFile = confPath + "/" + fname;
- }
-
- // Make sure the file exists, is readable and is not a hidden file
- if (!zkClient.exists(adminFile, true)) {
- rsp.setException(new SolrException(ErrorCode.NOT_FOUND, "Can not find: "
- + adminFile));
+
+ String adminFile = getAdminFileFromZooKeeper(req, rsp, zkClient, hiddenFiles);
+
+ if (adminFile == null) {
return;
}
-
+
// Show a directory listing
List<String> children = zkClient.getChildren(adminFile, null, true);
if (children.size() > 0) {
NamedList<SimpleOrderedMap<Object>> files = new SimpleOrderedMap<SimpleOrderedMap<Object>>();
for (String f : children) {
- if (hiddenFiles.contains(f.toUpperCase(Locale.ROOT))) {
- continue; // don't show 'hidden' files
- }
- if (f.startsWith(".")) {
- continue; // skip hidden system files...
+ if (isHiddenFile(req, rsp, f, false, hiddenFiles)) {
+ continue;
}
-
+
SimpleOrderedMap<Object> fileInfo = new SimpleOrderedMap<Object>();
files.add(f, fileInfo);
- List<String> fchildren = zkClient.getChildren(adminFile, null, true);
+ List<String> fchildren = zkClient.getChildren(adminFile + "/" + f, null, true);
if (fchildren.size() > 0) {
fileInfo.add("directory", true);
} else {
@@ -199,45 +188,24 @@ public class ShowFileRequestHandler exte
rsp.setHttpCaching(false);
}
+ // Return the file indicated (or the directory listing) from the local file system.
private void showFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) {
- File adminFile = null;
-
- final SolrResourceLoader loader = req.getCore().getResourceLoader();
- File configdir = new File( loader.getConfigDir() );
- if (!configdir.exists()) {
- // TODO: maybe we should just open it this way to start with?
- try {
- configdir = new File( loader.getClassLoader().getResource(loader.getConfigDir()).toURI() );
- } catch (URISyntaxException e) {
- rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Can not access configuration directory!", e));
- return;
- }
- }
- String fname = req.getParams().get("file", null);
- if( fname == null ) {
- adminFile = configdir;
- }
- else {
- fname = fname.replace( '\\', '/' ); // normalize slashes
- if( hiddenFiles.contains( fname.toUpperCase(Locale.ROOT) ) ) {
- rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Can not access: "+fname ));
- return;
- }
- if( fname.indexOf( ".." ) >= 0 ) {
- rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Invalid path: "+fname ));
- return;
- }
- adminFile = new File( configdir, fname );
+ File adminFile = getAdminFileFromFileSystem(req, rsp, hiddenFiles);
+
+ if (adminFile == null) { // exception already recorded
+ return;
}
-
+
// Make sure the file exists, is readable and is not a hidden file
if( !adminFile.exists() ) {
+ log.error("Can not find: "+adminFile.getName() + " ["+adminFile.getAbsolutePath()+"]");
rsp.setException(new SolrException
( ErrorCode.NOT_FOUND, "Can not find: "+adminFile.getName()
+ " ["+adminFile.getAbsolutePath()+"]" ));
return;
}
if( !adminFile.canRead() || adminFile.isHidden() ) {
+ log.error("Can not show: "+adminFile.getName() + " ["+adminFile.getAbsolutePath()+"]");
rsp.setException(new SolrException
( ErrorCode.NOT_FOUND, "Can not show: "+adminFile.getName()
+ " ["+adminFile.getAbsolutePath()+"]" ));
@@ -246,19 +214,17 @@ public class ShowFileRequestHandler exte
// Show a directory listing
if( adminFile.isDirectory() ) {
-
- int basePath = configdir.getAbsolutePath().length() + 1;
+ // it's really a directory, just go for it.
+ int basePath = adminFile.getAbsolutePath().length() + 1;
NamedList<SimpleOrderedMap<Object>> files = new SimpleOrderedMap<SimpleOrderedMap<Object>>();
for( File f : adminFile.listFiles() ) {
String path = f.getAbsolutePath().substring( basePath );
path = path.replace( '\\', '/' ); // normalize slashes
- if( hiddenFiles.contains( path.toUpperCase(Locale.ROOT) ) ) {
- continue; // don't show 'hidden' files
- }
- if( f.isHidden() || f.getName().startsWith( "." ) ) {
- continue; // skip hidden system files...
+
+ if (isHiddenFile(req, rsp, f.getName().replace('\\', '/'), false, hiddenFiles)) {
+ continue;
}
-
+
SimpleOrderedMap<Object> fileInfo = new SimpleOrderedMap<Object>();
files.add( path, fileInfo );
if( f.isDirectory() ) {
@@ -270,7 +236,7 @@ public class ShowFileRequestHandler exte
}
fileInfo.add( "modified", new Date( f.lastModified() ) );
}
- rsp.add( "files", files );
+ rsp.add("files", files);
}
else {
// Include the file contents
@@ -280,19 +246,139 @@ public class ShowFileRequestHandler exte
req.setParams(params);
ContentStreamBase content = new ContentStreamBase.FileStream( adminFile );
- content.setContentType( req.getParams().get( USE_CONTENT_TYPE ) );
+ content.setContentType(req.getParams().get(USE_CONTENT_TYPE));
rsp.add(RawResponseWriter.CONTENT, content);
}
rsp.setHttpCaching(false);
}
-
-
+
+ //////////////////////// Static methods //////////////////////////////
+
+ public static boolean isHiddenFile(SolrQueryRequest req, SolrQueryResponse rsp, String fnameIn, boolean reportError,
+ Set<String> hiddenFiles) {
+ String fname = fnameIn.toUpperCase(Locale.ROOT);
+ if (hiddenFiles.contains(fname) || hiddenFiles.contains("*")) {
+ if (reportError) {
+ log.error("Cannot access " + fname);
+ rsp.setException(new SolrException(SolrException.ErrorCode.FORBIDDEN, "Can not access: " + fnameIn));
+ }
+ return true;
+ }
+
+ // This is slightly off, a valid path is something like ./schema.xml. I don't think it's worth the effort though
+ // to fix it to handle all possibilities though.
+ if (fname.indexOf("..") >= 0 || fname.startsWith(".")) {
+ if (reportError) {
+ log.error("Invalid path: " + fname);
+ rsp.setException(new SolrException(SolrException.ErrorCode.FORBIDDEN, "Invalid path: " + fnameIn));
+ }
+ return true;
+ }
+
+ // Make sure that if the schema is managed, we don't allow editing. Don't really want to put
+ // this in the init since we're not entirely sure when the managed schema will get initialized relative to this
+ // handler.
+ SolrCore core = req.getCore();
+ IndexSchema schema = core.getLatestSchema();
+ if (schema instanceof ManagedIndexSchema) {
+ String managed = schema.getResourceName();
+
+ if (fname.equalsIgnoreCase(managed)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Refactored to be usable from multiple methods. Gets the path of the requested file from ZK.
+ // Returns null if the file is not found.
+ //
+ // Assumes that the file is in a parameter called "file".
+
+ public static String getAdminFileFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp, SolrZkClient zkClient,
+ Set<String> hiddenFiles)
+ throws KeeperException, InterruptedException {
+ String adminFile = null;
+ SolrCore core = req.getCore();
+
+ final ZkSolrResourceLoader loader = (ZkSolrResourceLoader) core
+ .getResourceLoader();
+ String confPath = loader.getCollectionZkPath();
+
+ String fname = req.getParams().get("file", null);
+ if (fname == null) {
+ adminFile = confPath;
+ } else {
+ fname = fname.replace('\\', '/'); // normalize slashes
+ if (isHiddenFile(req, rsp, fname, true, hiddenFiles)) {
+ return null;
+ }
+ if (fname.startsWith("/")) { // Only files relative to conf are valid
+ fname = fname.substring(1);
+ }
+ adminFile = confPath + "/" + fname;
+ }
+
+ // Make sure the file exists, is readable and is not a hidden file
+ if (!zkClient.exists(adminFile, true)) {
+ log.error("Can not find: " + adminFile);
+ rsp.setException(new SolrException(SolrException.ErrorCode.NOT_FOUND, "Can not find: "
+ + adminFile));
+ return null;
+ }
+
+ return adminFile;
+ }
+
+
+ // Find the file indicated by the "file=XXX" parameter or the root of the conf directory on the local
+ // file system. Respects all the "interesting" stuff around what the resource loader does to find files.
+ public static File getAdminFileFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp,
+ Set<String> hiddenFiles) {
+ File adminFile = null;
+ final SolrResourceLoader loader = req.getCore().getResourceLoader();
+ File configdir = new File( loader.getConfigDir() );
+ if (!configdir.exists()) {
+ // TODO: maybe we should just open it this way to start with?
+ try {
+ configdir = new File( loader.getClassLoader().getResource(loader.getConfigDir()).toURI() );
+ } catch (URISyntaxException e) {
+ log.error("Can not access configuration directory!");
+ rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Can not access configuration directory!", e));
+ return null;
+ }
+ }
+ String fname = req.getParams().get("file", null);
+ if( fname == null ) {
+ adminFile = configdir;
+ }
+ else {
+ fname = fname.replace( '\\', '/' ); // normalize slashes
+ if( hiddenFiles.contains( fname.toUpperCase(Locale.ROOT) ) ) {
+ log.error("Can not access: "+ fname);
+ rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Can not access: "+fname ));
+ return null;
+ }
+ if( fname.indexOf( ".." ) >= 0 ) {
+ log.error("Invalid path: "+ fname);
+ rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Invalid path: "+fname ));
+ return null;
+ }
+ adminFile = new File( configdir, fname );
+ }
+ return adminFile;
+ }
+
+ public final Set<String> getHiddenFiles() {
+ return hiddenFiles;
+ }
+
//////////////////////// SolrInfoMBeans methods //////////////////////
@Override
public String getDescription() {
- return "Admin Get File -- view config files directly";
+ return "Admin Config File -- view or update config files directly";
}
@Override
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java Thu Dec 19 17:48:47 2013
@@ -48,6 +48,7 @@ public class FieldFacetStats {
public final String name;
final SchemaField facet_sf;
final SchemaField field_sf;
+ final boolean calcDistinct;
public final Map<String, StatsValues> facetStatsValues;
@@ -63,10 +64,11 @@ public class FieldFacetStats {
private final BytesRef tempBR = new BytesRef();
- public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf) {
+ public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf, boolean calcDistinct) {
this.name = name;
this.field_sf = field_sf;
this.facet_sf = facet_sf;
+ this.calcDistinct = calcDistinct;
topLevelReader = searcher.getAtomicReader();
valueSource = facet_sf.getType().getValueSource(facet_sf, null);
@@ -78,7 +80,7 @@ public class FieldFacetStats {
private StatsValues getStatsValues(String key) throws IOException {
StatsValues stats = facetStatsValues.get(key);
if (stats == null) {
- stats = StatsValuesFactory.createStatsValues(field_sf);
+ stats = StatsValuesFactory.createStatsValues(field_sf, calcDistinct);
facetStatsValues.put(key, stats);
stats.setNextReader(context);
}
@@ -139,7 +141,7 @@ public class FieldFacetStats {
String key = (String) pairs.getKey();
StatsValues facetStats = facetStatsValues.get(key);
if (facetStats == null) {
- facetStats = StatsValuesFactory.createStatsValues(field_sf);
+ facetStats = StatsValuesFactory.createStatsValues(field_sf, calcDistinct);
facetStatsValues.put(key, facetStats);
}
Integer count = (Integer) pairs.getValue();
Modified: lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java?rev=1552377&r1=1552376&r2=1552377&view=diff
==============================================================================
--- lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene5339/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java Thu Dec 19 17:48:47 2013
@@ -16,19 +16,6 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -44,6 +31,18 @@ import org.apache.solr.util.DefaultSolrT
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized {
protected static Logger log = LoggerFactory.getLogger(HttpShardHandlerFactory.class);
@@ -158,12 +157,7 @@ public class HttpShardHandlerFactory ext
}
protected LBHttpSolrServer createLoadbalancer(HttpClient httpClient){
- try {
- return new LBHttpSolrServer(httpClient);
- } catch (MalformedURLException e) {
- // should be impossible since we're not passing any URLs here
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
+ return new LBHttpSolrServer(httpClient);
}
protected <T> T getParameter(NamedList initArgs, String configKey, T defaultValue) {