You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/10/21 20:58:44 UTC
svn commit: r1534320 [30/39] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/lucene/expressions/
dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/
dev-tools/maven/lucene/ dev-tools/maven/lucene/expressions/...
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java Mon Oct 21 18:58:24 2013
@@ -3,6 +3,7 @@ package org.apache.solr.cloud;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -19,7 +20,9 @@ import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.List;
import java.util.concurrent.TimeoutException;
@@ -44,6 +47,7 @@ public class ZkCLI {
private static final String MAKEPATH = "makepath";
private static final String PUT = "put";
+ private static final String PUT_FILE = "putfile";
private static final String DOWNCONFIG = "downconfig";
private static final String ZK_CLI_NAME = "ZkCLI";
private static final String HELP = "help";
@@ -87,7 +91,8 @@ public class ZkCLI {
.hasArg(true)
.withDescription(
"cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG
- + ", " + LINKCONFIG + ", " + MAKEPATH + ", "+ PUT + ", "+ LIST + ", " + CLEAR).create(CMD));
+ + ", " + LINKCONFIG + ", " + MAKEPATH + ", " + PUT + ", " + PUT_FILE + ","
+ + LIST + ", " + CLEAR).create(CMD));
Option zkHostOption = new Option("z", ZKHOST, true,
"ZooKeeper host address");
@@ -131,6 +136,7 @@ public class ZkCLI {
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LINKCONFIG + " -" + COLLECTION + " collection1" + " -" + CONFNAME + " myconf");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'");
+ System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT_FILE + " /solr.xml /User/myuser/solr/solr.xml");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
return;
@@ -244,6 +250,20 @@ public class ZkCLI {
}
zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes("UTF-8"),
acl, CreateMode.PERSISTENT, true);
+ } else if (line.getOptionValue(CMD).equals(PUT_FILE)) {
+ List arglist = line.getArgList();
+ if (arglist.size() != 2) {
+ System.out.println("-" + PUT_FILE + " requires two args - the path to create in ZK and the path to the local file");
+ System.exit(1);
+ }
+ InputStream is = new FileInputStream(arglist.get(1).toString());
+ try {
+ zkClient.create(arglist.get(0).toString(), IOUtils.toByteArray(is),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, true);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+
}
} finally {
if (solrPort != null) {
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkController.java Mon Oct 21 18:58:24 2013
@@ -102,8 +102,43 @@ public final class ZkController {
public final static String COLLECTION_PARAM_PREFIX="collection.";
public final static String CONFIGNAME_PROP="configName";
-
- private final Map<String, ElectionContext> electionContexts = Collections.synchronizedMap(new HashMap<String, ElectionContext>());
+ static class ContextKey {
+
+ private String collection;
+ private String coreNodeName;
+
+ public ContextKey(String collection, String coreNodeName) {
+ this.collection = collection;
+ this.coreNodeName = coreNodeName;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + ((collection == null) ? 0 : collection.hashCode());
+ result = prime * result
+ + ((coreNodeName == null) ? 0 : coreNodeName.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+ ContextKey other = (ContextKey) obj;
+ if (collection == null) {
+ if (other.collection != null) return false;
+ } else if (!collection.equals(other.collection)) return false;
+ if (coreNodeName == null) {
+ if (other.coreNodeName != null) return false;
+ } else if (!coreNodeName.equals(other.coreNodeName)) return false;
+ return true;
+ }
+ }
+ private final Map<ContextKey, ElectionContext> electionContexts = Collections.synchronizedMap(new HashMap<ContextKey, ElectionContext>());
private SolrZkClient zkClient;
private ZkCmdExecutor cmdExecutor;
@@ -930,7 +965,7 @@ public final class ZkController {
collection, coreNodeName, ourProps, this, cc);
leaderElector.setup(context);
- electionContexts.put(coreNodeName, context);
+ electionContexts.put(new ContextKey(collection, coreNodeName), context);
leaderElector.joinElection(context, false);
}
@@ -1001,6 +1036,7 @@ public final class ZkController {
ZkStateReader.SHARD_ID_PROP, cd.getCloudDescriptor().getShardId(),
ZkStateReader.SHARD_RANGE_PROP, cd.getCloudDescriptor().getShardRange(),
ZkStateReader.SHARD_STATE_PROP, cd.getCloudDescriptor().getShardState(),
+ ZkStateReader.SHARD_PARENT_PROP, cd.getCloudDescriptor().getShardParent(),
ZkStateReader.COLLECTION_PROP, cd.getCloudDescriptor()
.getCollectionName(),
ZkStateReader.NUM_SHARDS_PROP, numShards != null ? numShards.toString()
@@ -1030,13 +1066,14 @@ public final class ZkController {
public void unregister(String coreName, CoreDescriptor cd)
throws InterruptedException, KeeperException {
final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
- ElectionContext context = electionContexts.remove(coreNodeName);
-
- assert context != null : coreNodeName;
+ final String collection = cd.getCloudDescriptor().getCollectionName();
+ assert collection != null;
+ ElectionContext context = electionContexts.remove(new ContextKey(collection, coreNodeName));
if (context != null) {
context.cancelElection();
}
+
CloudDescriptor cloudDescriptor = cd.getCloudDescriptor();
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
@@ -1124,18 +1161,6 @@ public final class ZkController {
getConfName(collection, collectionPath, collectionProps);
}
- if (collectionProps.get(DocCollection.DOC_ROUTER) == null) {
- Object numShards = collectionProps.get(ZkStateReader.NUM_SHARDS_PROP);
- if (numShards == null) {
- numShards = System.getProperty(ZkStateReader.NUM_SHARDS_PROP);
- }
- if (numShards == null) {
- collectionProps.put(DocCollection.DOC_ROUTER, ImplicitDocRouter.NAME);
- } else {
- collectionProps.put(DocCollection.DOC_ROUTER, DocRouter.DEFAULT_NAME);
- }
- }
-
collectionProps.remove(ZkStateReader.NUM_SHARDS_PROP); // we don't put numShards in the collections properties
ZkNodeProps zkProps = new ZkNodeProps(collectionProps);
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java Mon Oct 21 18:58:24 2013
@@ -18,6 +18,7 @@ package org.apache.solr.cloud;
*/
import java.io.ByteArrayInputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
@@ -75,7 +76,7 @@ public class ZkSolrResourceLoader extend
String file = collectionZkPath + "/" + resource;
try {
if (zkController.pathExists(file)) {
- byte[] bytes = zkController.getZkClient().getData(collectionZkPath + "/" + resource, null, null, true);
+ byte[] bytes = zkController.getZkClient().getData(file, null, null, true);
return new ByteArrayInputStream(bytes);
}
} catch (Exception e) {
@@ -83,7 +84,7 @@ public class ZkSolrResourceLoader extend
}
try {
// delegate to the class loader (looking into $INSTANCE_DIR/lib jars)
- is = classLoader.getResourceAsStream(resource);
+ is = classLoader.getResourceAsStream(resource.replace(File.separatorChar, '/'));
} catch (Exception e) {
throw new IOException("Error opening " + resource, e);
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolr.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolr.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolr.java Mon Oct 21 18:58:24 2013
@@ -55,16 +55,13 @@ public abstract class ConfigSolr {
try {
if (!configFile.exists()) {
- log.info("{} does not exist, using default configuration", configFile.getAbsolutePath());
- inputStream = new ByteArrayInputStream(ConfigSolrXmlOld.DEF_SOLR_XML.getBytes(Charsets.UTF_8));
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "solr.xml does not exist in " + configFile.getAbsolutePath() + " cannot start Solr");
}
else {
inputStream = new FileInputStream(configFile);
}
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ByteStreams.copy(inputStream, baos);
- String originalXml = IOUtils.toString(new ByteArrayInputStream(baos.toByteArray()), "UTF-8");
- return fromInputStream(loader, new ByteArrayInputStream(baos.toByteArray()), configFile, originalXml);
+ return fromInputStream(loader, inputStream);
}
catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@@ -76,13 +73,17 @@ public abstract class ConfigSolr {
}
public static ConfigSolr fromString(String xml) {
- return fromInputStream(null, new ByteArrayInputStream(xml.getBytes(Charsets.UTF_8)), null, xml);
+ return fromInputStream(null, new ByteArrayInputStream(xml.getBytes(Charsets.UTF_8)));
}
- public static ConfigSolr fromInputStream(SolrResourceLoader loader, InputStream is, File file, String originalXml) {
+ public static ConfigSolr fromInputStream(SolrResourceLoader loader, InputStream is) {
try {
- Config config = new Config(loader, null, new InputSource(is), null, false);
- return fromConfig(config, file, originalXml);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ByteStreams.copy(is, baos);
+ String originalXml = IOUtils.toString(new ByteArrayInputStream(baos.toByteArray()), "UTF-8");
+ ByteArrayInputStream dup = new ByteArrayInputStream(baos.toByteArray());
+ Config config = new Config(loader, null, new InputSource(dup), null, false);
+ return fromConfig(config, originalXml);
}
catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@@ -93,9 +94,9 @@ public abstract class ConfigSolr {
return fromFile(loader, new File(solrHome, SOLR_XML_FILE));
}
- public static ConfigSolr fromConfig(Config config, File file, String originalXml) {
+ public static ConfigSolr fromConfig(Config config, String originalXml) {
boolean oldStyle = (config.getNode("solr/cores", false) != null);
- return oldStyle ? new ConfigSolrXmlOld(config, file, originalXml)
+ return oldStyle ? new ConfigSolrXmlOld(config, originalXml)
: new ConfigSolrXml(config);
}
@@ -188,7 +189,7 @@ public abstract class ConfigSolr {
public LogWatcherConfig getLogWatcherConfig() {
return new LogWatcherConfig(
- getBool(CfgProp.SOLR_LOGGING_ENABLED, false),
+ getBool(CfgProp.SOLR_LOGGING_ENABLED, true),
get(CfgProp.SOLR_LOGGING_CLASS, null),
get(CfgProp.SOLR_LOGGING_WATCHER_THRESHOLD, null),
getInt(CfgProp.SOLR_LOGGING_WATCHER_SIZE, 50)
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Mon Oct 21 18:58:24 2013
@@ -50,20 +50,22 @@ public class ConfigSolrXmlOld extends Co
private final CoresLocator persistor;
+ public static final String DEFAULT_DEFAULT_CORE_NAME = "collection1";
+
@Override
protected String getShardHandlerFactoryConfigPath() {
return "solr/cores/shardHandlerFactory";
}
- public ConfigSolrXmlOld(Config config, File configFile, String originalXML) {
+ public ConfigSolrXmlOld(Config config, String originalXML) {
super(config);
try {
checkForIllegalConfig();
fillPropMap();
config.substituteProperties();
initCoreList();
- this.persistor = isPersistent() ? new SolrXMLCoresLocator(configFile, originalXML, this)
- : new SolrXMLCoresLocator.NonPersistingLocator(configFile, originalXML, this);
+ this.persistor = isPersistent() ? new SolrXMLCoresLocator(originalXML, this)
+ : new SolrXMLCoresLocator.NonPersistingLocator(originalXML, this);
}
catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@@ -273,19 +275,4 @@ public class ConfigSolrXmlOld extends Co
}
return new Properties();
}
-
- public static final String DEFAULT_DEFAULT_CORE_NAME = "collection1";
-
- public static final String DEF_SOLR_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
- + "<solr persistent=\"false\">\n"
- + " <cores adminPath=\"/admin/cores\" defaultCoreName=\""
- + DEFAULT_DEFAULT_CORE_NAME
- + "\""
- + " host=\"${host:}\" hostPort=\"${hostPort:}\" hostContext=\"${hostContext:}\" zkClientTimeout=\"${zkClientTimeout:15000}\""
- + ">\n"
- + " <core name=\""
- + DEFAULT_DEFAULT_CORE_NAME
- + "\" shard=\"${shard:}\" collection=\"${collection:collection1}\" instanceDir=\"collection1\" />\n"
- + " </cores>\n" + "</solr>";
-
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreContainer.java Mon Oct 21 18:58:24 2013
@@ -18,18 +18,19 @@
package org.apache.solr.core;
import com.google.common.collect.Maps;
+
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.admin.InfoHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.logging.LogWatcher;
-import org.apache.solr.logging.jul.JulWatcher;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.util.DefaultSolrThreadFactory;
@@ -37,6 +38,7 @@ import org.apache.solr.util.FileUtils;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static com.google.common.base.Preconditions.checkNotNull;
import java.io.File;
import java.text.SimpleDateFormat;
@@ -59,7 +61,26 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import static com.google.common.base.Preconditions.checkNotNull;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.handler.admin.CoreAdminHandler;
+import org.apache.solr.handler.admin.InfoHandler;
+import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.logging.LogWatcher;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.IndexSchemaFactory;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.FileUtils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
/**
@@ -86,6 +107,9 @@ public class CoreContainer {
protected ZkContainer zkSys = new ZkContainer();
private ShardHandlerFactory shardHandlerFactory;
+
+ private ExecutorService updateExecutor = Executors.newCachedThreadPool(
+ new SolrjNamedThreadFactory("updateExecutor"));
protected LogWatcher logging = null;
@@ -193,7 +217,7 @@ public class CoreContainer {
solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
- logging = JulWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
+ logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
shareSchema = cfg.hasSchemaCache();
@@ -210,8 +234,10 @@ public class CoreContainer {
containerProperties = cfg.getSolrProperties("solr");
// setup executor to load cores in parallel
- ExecutorService coreLoadExecutor = Executors.newFixedThreadPool(cfg.getCoreLoadThreadCount(),
- new DefaultSolrThreadFactory("coreLoadExecutor"));
+ // do not limit the size of the executor in zk mode since cores may try and wait for each other.
+ ExecutorService coreLoadExecutor = Executors.newFixedThreadPool(
+ ( zkSys.getZkController() == null ? cfg.getCoreLoadThreadCount() : Integer.MAX_VALUE ),
+ new DefaultSolrThreadFactory("coreLoadExecutor") );
try {
CompletionService<SolrCore> completionService = new ExecutorCompletionService<SolrCore>(
@@ -377,6 +403,8 @@ public class CoreContainer {
shardHandlerFactory.close();
}
+ ExecutorUtil.shutdownAndAwaitTermination(updateExecutor);
+
// we want to close zk stuff last
zkSys.close();
@@ -480,12 +508,14 @@ public class CoreContainer {
SolrResourceLoader solrLoader = null;
SolrConfig config = null;
- solrLoader = new SolrResourceLoader(instanceDir, loader.getClassLoader(), dcore.getCoreProperties());
+ solrLoader = new SolrResourceLoader(instanceDir, loader.getClassLoader(), dcore.getSubstitutableProperties());
try {
config = new SolrConfig(solrLoader, dcore.getConfigName(), null);
} catch (Exception e) {
log.error("Failed to load file {}", new File(instanceDir, dcore.getConfigName()).getAbsolutePath());
- throw new SolrException(ErrorCode.SERVER_ERROR, "Could not load config for " + dcore.getConfigName(), e);
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "Could not load config file " + new File(instanceDir, dcore.getConfigName()).getAbsolutePath(),
+ e);
}
IndexSchema schema = null;
@@ -646,7 +676,7 @@ public class CoreContainer {
SolrResourceLoader solrLoader;
if(zkSys.getZkController() == null) {
solrLoader = new SolrResourceLoader(instanceDir.getAbsolutePath(), loader.getClassLoader(),
- cd.getCoreProperties());
+ cd.getSubstitutableProperties());
} else {
try {
String collection = cd.getCloudDescriptor().getCollectionName();
@@ -659,7 +689,7 @@ public class CoreContainer {
"Could not find config name for collection:" + collection);
}
solrLoader = new ZkSolrResourceLoader(instanceDir.getAbsolutePath(), zkConfigName, loader.getClassLoader(),
- cd.getCoreProperties(), zkSys.getZkController());
+ cd.getSubstitutableProperties(), zkSys.getZkController());
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
@@ -920,6 +950,10 @@ public class CoreContainer {
return shardHandlerFactory;
}
+ public ExecutorService getUpdateExecutor() {
+ return updateExecutor;
+ }
+
// Just to tidy up the code where it did this in-line.
private SolrException recordAndThrow(String name, String msg, Exception ex) {
synchronized (coreInitFailures) {
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java Mon Oct 21 18:58:24 2013
@@ -19,9 +19,11 @@ package org.apache.solr.core;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+
import org.apache.commons.lang.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.util.IOUtils;
import org.apache.solr.util.PropertiesUtil;
@@ -56,6 +58,7 @@ public class CoreDescriptor {
public static final String CORE_LOADONSTARTUP = "loadOnStartup";
public static final String CORE_TRANSIENT = "transient";
public static final String CORE_NODE_NAME = "coreNodeName";
+ public static final String SOLR_CORE_PROP_PREFIX = "solr.core.";
public static final String DEFAULT_EXTERNAL_PROPERTIES_FILE = "conf" + File.separator + "solrcore.properties";
@@ -119,6 +122,9 @@ public class CoreDescriptor {
/** The properties for this core, as available through getProperty() */
protected final Properties coreProperties = new Properties();
+ /** The properties for this core, substitutable by resource loaders */
+ protected final Properties substitutableProperties = new Properties();
+
/**
* Create a new CoreDescriptor.
* @param container the CoreDescriptor's container
@@ -128,6 +134,19 @@ public class CoreDescriptor {
*/
public CoreDescriptor(CoreContainer container, String name, String instanceDir,
Properties coreProps) {
+ this(container, name, instanceDir, coreProps, null);
+ }
+
+ /**
+ * Create a new CoreDescriptor.
+ * @param container the CoreDescriptor's container
+ * @param name the CoreDescriptor's name
+ * @param instanceDir a String containing the instanceDir
+ * @param coreProps a Properties object of the properties for this core
+ * @param params additional params
+ */
+ public CoreDescriptor(CoreContainer container, String name, String instanceDir,
+ Properties coreProps, SolrParams params) {
this.coreContainer = container;
@@ -160,10 +179,14 @@ public class CoreDescriptor {
}
loadExtraProperties();
+ buildSubstitutableProperties();
// TODO maybe make this a CloudCoreDescriptor subclass?
if (container.isZooKeeperAware()) {
cloudDesc = new CloudDescriptor(name, coreProperties);
+ if (params != null) {
+ cloudDesc.setParams(params);
+ }
}
else {
cloudDesc = null;
@@ -201,6 +224,20 @@ public class CoreDescriptor {
}
}
+ /**
+ * Create the properties object used by resource loaders, etc, for property
+ * substitution. The default solr properties are prefixed with 'solr.core.', so,
+ * e.g., 'name' becomes 'solr.core.name'
+ */
+ protected void buildSubstitutableProperties() {
+ for (String propName : coreProperties.stringPropertyNames()) {
+ String propValue = coreProperties.getProperty(propName);
+ if (!isUserDefinedProperty(propName))
+ propName = SOLR_CORE_PROP_PREFIX + propName;
+ substitutableProperties.setProperty(propName, propValue);
+ }
+ }
+
protected File resolvePaths(String filepath) {
File file = new File(filepath);
if (file.isAbsolute())
@@ -243,12 +280,14 @@ public class CoreDescriptor {
*/
public CoreDescriptor(String coreName, CoreDescriptor other) {
this.coreContainer = other.coreContainer;
+ this.cloudDesc = other.cloudDesc;
this.originalExtraProperties.putAll(other.originalExtraProperties);
this.originalCoreProperties.putAll(other.originalCoreProperties);
this.coreProperties.putAll(other.coreProperties);
+ this.substitutableProperties.putAll(other.substitutableProperties);
this.coreProperties.setProperty(CORE_NAME, coreName);
this.originalCoreProperties.setProperty(CORE_NAME, coreName);
- this.cloudDesc = other.cloudDesc;
+ this.substitutableProperties.setProperty(SOLR_CORE_PROP_PREFIX + CORE_NAME, coreName);
}
public String getPropertiesName() {
@@ -336,11 +375,11 @@ public class CoreDescriptor {
}
/**
- * Returns all properties defined on this CoreDescriptor
- * @return all properties defined on this CoreDescriptor
+ * Returns all substitutable properties defined on this CoreDescriptor
+ * @return all substitutable properties defined on this CoreDescriptor
*/
- public Properties getCoreProperties() {
- return coreProperties;
+ public Properties getSubstitutableProperties() {
+ return substitutableProperties;
}
@Override
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java Mon Oct 21 18:58:24 2013
@@ -31,7 +31,6 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.util.Date;
import java.util.List;
import java.util.Properties;
@@ -78,6 +77,7 @@ public class CorePropertiesLocator imple
Properties p = buildCoreProperties(cd);
Writer os = null;
try {
+ propfile.getParentFile().mkdirs();
os = new OutputStreamWriter(new FileOutputStream(propfile), Charsets.UTF_8);
p.store(os, "Written by CorePropertiesLocator");
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java Mon Oct 21 18:58:24 2013
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.IOException;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
@@ -55,4 +56,21 @@ public abstract class IndexReaderFactory
*/
public abstract DirectoryReader newReader(Directory indexDir, SolrCore core)
throws IOException;
+
+ /**
+ * Creates a new IndexReader instance using the given IndexWriter.
+ * <p>
+ * This is used for opening the initial reader in NRT mode ({@code nrtMode=true}
+ * in solrconfig.xml)
+ *
+ * @param writer IndexWriter
+ * @param core {@link SolrCore} instance where this reader will be used. NOTE:
+ * this SolrCore instance may not be fully configured yet, but basic things like
+ * {@link SolrCore#getCoreDescriptor()}, {@link SolrCore#getLatestSchema()} and
+ * {@link SolrCore#getSolrConfig()} are valid.
+ * @return An IndexReader instance
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public abstract DirectoryReader newReader(IndexWriter writer, SolrCore core)
+ throws IOException;
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java Mon Oct 21 18:58:24 2013
@@ -3,7 +3,7 @@ package org.apache.solr.core;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.plugin.SolrCoreAware;
@@ -51,7 +51,7 @@ public class SchemaCodecFactory extends
@Override
public void init(NamedList args) {
super.init(args);
- codec = new Lucene42Codec() {
+ codec = new Lucene46Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
final SchemaField fieldOrNull = core.getLatestSchema().getFieldOrNull(field);
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrConfig.java Mon Oct 21 18:58:24 2013
@@ -17,6 +17,7 @@
package org.apache.solr.core;
+import static org.apache.solr.core.SolrConfig.PluginOpts.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.schema.IndexSchemaFactory;
@@ -70,7 +71,15 @@ public class SolrConfig extends Config {
public static final String DEFAULT_CONF_FILE = "solrconfig.xml";
-
+ static enum PluginOpts {
+ MULTI_OK,
+ REQUIRE_NAME,
+ REQUIRE_CLASS,
+ // EnumSet.of and/or EnumSet.copyOf(Collection) are anoying
+ // because of type determination
+ NOOP
+ }
+
/** Creates a default instance from the solrconfig.xml. */
public SolrConfig()
throws ParserConfigurationException, IOException, SAXException {
@@ -143,7 +152,7 @@ public class SolrConfig extends Config {
defaultIndexConfig = mainIndexConfig = null;
indexConfigPrefix = "indexConfig";
}
- reopenReaders = getBool(indexConfigPrefix+"/reopenReaders", true);
+ nrtMode = getBool(indexConfigPrefix+"/nrtMode", true);
// Parse indexConfig section, using mainIndex as backup in case old config is used
indexConfig = new SolrIndexConfig(this, "indexConfig", mainIndexConfig);
@@ -207,27 +216,46 @@ public class SolrConfig extends Config {
}
maxWarmingSearchers = getInt("query/maxWarmingSearchers",Integer.MAX_VALUE);
- loadPluginInfo(SolrRequestHandler.class,"requestHandler",true, true);
- loadPluginInfo(QParserPlugin.class,"queryParser",true, true);
- loadPluginInfo(QueryResponseWriter.class,"queryResponseWriter",true, true);
- loadPluginInfo(ValueSourceParser.class,"valueSourceParser",true, true);
- loadPluginInfo(TransformerFactory.class,"transformer",true, true);
- loadPluginInfo(SearchComponent.class,"searchComponent",true, true);
- loadPluginInfo(QueryConverter.class,"queryConverter",true, true);
+ loadPluginInfo(SolrRequestHandler.class,"requestHandler",
+ REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
+ loadPluginInfo(QParserPlugin.class,"queryParser",
+ REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
+ loadPluginInfo(QueryResponseWriter.class,"queryResponseWriter",
+ REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
+ loadPluginInfo(ValueSourceParser.class,"valueSourceParser",
+ REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
+ loadPluginInfo(TransformerFactory.class,"transformer",
+ REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
+ loadPluginInfo(SearchComponent.class,"searchComponent",
+ REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
+
+ // TODO: WTF is up with queryConverter???
+ // it aparently *only* works as a singleton? - SOLR-4304
+ // and even then -- only if there is a single SpellCheckComponent
+ // because of queryConverter.setAnalyzer
+ loadPluginInfo(QueryConverter.class,"queryConverter",
+ REQUIRE_NAME, REQUIRE_CLASS);
// this is hackish, since it picks up all SolrEventListeners,
// regardless of when/how/why they are used (or even if they are
// declared outside of the appropriate context) but there's no nice
// way around that in the PluginInfo framework
- loadPluginInfo(SolrEventListener.class, "//listener",false, true);
+ loadPluginInfo(SolrEventListener.class, "//listener",
+ REQUIRE_CLASS, MULTI_OK);
- loadPluginInfo(DirectoryFactory.class,"directoryFactory",false, true);
- loadPluginInfo(IndexDeletionPolicy.class,indexConfigPrefix+"/deletionPolicy",false, true);
- loadPluginInfo(CodecFactory.class,"codecFactory",false, false);
- loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",false, true);
- loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",false, false);
- loadPluginInfo(UpdateLog.class,"updateHandler/updateLog",false, false);
- loadPluginInfo(IndexSchemaFactory.class,"schemaFactory",false, true);
+ loadPluginInfo(DirectoryFactory.class,"directoryFactory",
+ REQUIRE_CLASS);
+ loadPluginInfo(IndexDeletionPolicy.class,indexConfigPrefix+"/deletionPolicy",
+ REQUIRE_CLASS);
+ loadPluginInfo(CodecFactory.class,"codecFactory",
+ REQUIRE_CLASS);
+ loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",
+ REQUIRE_CLASS);
+ loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",
+ MULTI_OK);
+ loadPluginInfo(UpdateLog.class,"updateHandler/updateLog");
+ loadPluginInfo(IndexSchemaFactory.class,"schemaFactory",
+ REQUIRE_CLASS);
updateHandlerInfo = loadUpdatehandlerInfo();
@@ -245,8 +273,19 @@ public class SolrConfig extends Config {
getBool("updateHandler/commitWithin/softCommit",true));
}
- private void loadPluginInfo(Class clazz, String tag, boolean requireName, boolean requireClass) {
+ private void loadPluginInfo(Class clazz, String tag, PluginOpts... opts) {
+ EnumSet<PluginOpts> options = EnumSet.<PluginOpts>of(NOOP, opts);
+ boolean requireName = options.contains(REQUIRE_NAME);
+ boolean requireClass = options.contains(REQUIRE_CLASS);
+
List<PluginInfo> result = readPluginInfos(tag, requireName, requireClass);
+
+ if (1 < result.size() && ! options.contains(MULTI_OK)) {
+ throw new SolrException
+ (SolrException.ErrorCode.SERVER_ERROR,
+ "Found " + result.size() + " configuration sections when at most "
+ + "1 is allowed matching expression: " + tag);
+ }
if(!result.isEmpty()) pluginStore.put(clazz.getName(),result);
}
@@ -277,7 +316,7 @@ public class SolrConfig extends Config {
public final int queryResultWindowSize;
public final int queryResultMaxDocsCached;
public final boolean enableLazyFieldLoading;
- public final boolean reopenReaders;
+ public final boolean nrtMode;
// DocSet
public final float hashSetInverseLoadFactor;
public final int hashDocSetMaxSize;
@@ -446,7 +485,15 @@ public class SolrConfig extends Config {
}
public PluginInfo getPluginInfo(String type){
List<PluginInfo> result = pluginStore.get(type);
- return result == null || result.isEmpty() ? null: result.get(0);
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ if (1 == result.size()) {
+ return result.get(0);
+ }
+
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Multiple plugins configured for type: " + type);
}
private void initLibs() {
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrCore.java Mon Oct 21 18:58:24 2013
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriter;
@@ -786,28 +787,24 @@ public final class SolrCore implements S
iwRef = prev.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
if (iwRef != null) {
final IndexWriter iw = iwRef.get();
+ final SolrCore core = this;
newReaderCreator = new Callable<DirectoryReader>() {
+ // this is used during a core reload
+
@Override
public DirectoryReader call() throws Exception {
- return DirectoryReader.open(iw, true);
+ if(getSolrConfig().nrtMode) {
+ // if in NRT mode, need to open from the previous writer
+ return indexReaderFactory.newReader(iw, core);
+ } else {
+ // if not NRT, need to create a new reader from the directory
+ return indexReaderFactory.newReader(iw.getDirectory(), core);
+ }
}
};
}
}
- // Open the searcher *before* the update handler so we don't end up
- // opening
- // one in the middle.
- // With lockless commits in Lucene now, this probably shouldn't be an
- // issue anymore
-
- try {
- getSearcher(false, false, null, true);
- } finally {
- newReaderCreator = null;
- if (iwRef != null) iwRef.decref();
- }
-
String updateHandlerClass = solrConfig.getUpdateHandlerInfo().className;
if (updateHandler == null) {
@@ -819,6 +816,13 @@ public final class SolrCore implements S
: updateHandlerClass, updateHandler);
}
infoRegistry.put("updateHandler", this.updateHandler);
+
+ try {
+ getSearcher(false, false, null, true);
+ } finally {
+ newReaderCreator = null;
+ if (iwRef != null) iwRef.decref();
+ }
// Finally tell anyone who wants to know
resourceLoader.inform(resourceLoader);
@@ -852,7 +856,7 @@ public final class SolrCore implements S
cd.getCloudDescriptor().setShardState(null);
cd.getCloudDescriptor().setShardRange(null);
-
+ cd.getCloudDescriptor().setShardParent(null);
}
// For debugging
// numOpens.incrementAndGet();
@@ -904,6 +908,7 @@ public final class SolrCore implements S
def = map.get(null);
}
if (def == null) {
+ log.info("no updateRequestProcessorChain defined as default, creating implicit default");
// construct the default chain
UpdateRequestProcessorFactory[] factories = new UpdateRequestProcessorFactory[]{
new LogUpdateProcessorFactory(),
@@ -1360,7 +1365,7 @@ public final class SolrCore implements S
SolrIndexSearcher tmp;
RefCounted<SolrIndexSearcher> newestSearcher = null;
- boolean nrt = solrConfig.reopenReaders && updateHandlerReopens;
+ boolean nrt = solrConfig.nrtMode && updateHandlerReopens;
openSearcherLock.lock();
try {
@@ -1381,8 +1386,7 @@ public final class SolrCore implements S
}
}
- if (newestSearcher != null && solrConfig.reopenReaders
- && (nrt || indexDirFile.equals(newIndexDirFile))) {
+ if (newestSearcher != null && (nrt || indexDirFile.equals(newIndexDirFile))) {
DirectoryReader newReader;
DirectoryReader currentReader = newestSearcher.get().getIndexReader();
@@ -1392,13 +1396,13 @@ public final class SolrCore implements S
RefCounted<IndexWriter> writer = getUpdateHandler().getSolrCoreState()
.getIndexWriter(null);
try {
- if (writer != null) {
- newReader = DirectoryReader.openIfChanged(currentReader,
- writer.get(), true);
+ if (writer != null && solrConfig.nrtMode) {
+ // if in NRT mode, open from the writer
+ newReader = DirectoryReader.openIfChanged(currentReader, writer.get(), true);
} else {
// verbose("start reopen without writer, reader=", currentReader);
+ // if not in NRT mode, just re-open the reader
newReader = DirectoryReader.openIfChanged(currentReader);
-
// verbose("reopen result", newReader);
}
} finally {
@@ -1432,6 +1436,16 @@ public final class SolrCore implements S
DirectoryReader newReader = newReaderCreator.call();
tmp = new SolrIndexSearcher(this, newIndexDir, getLatestSchema(), getSolrConfig().indexConfig,
(realtime ? "realtime":"main"), newReader, true, !realtime, true, directoryFactory);
+ } else if (solrConfig.nrtMode) {
+ RefCounted<IndexWriter> writer = getUpdateHandler().getSolrCoreState().getIndexWriter(this);
+ DirectoryReader newReader = null;
+ try {
+ newReader = indexReaderFactory.newReader(writer.get(), this);
+ } finally {
+ writer.decref();
+ }
+ tmp = new SolrIndexSearcher(this, newIndexDir, getLatestSchema(), getSolrConfig().indexConfig,
+ (realtime ? "realtime":"main"), newReader, true, !realtime, true, directoryFactory);
} else {
// normal open that happens at startup
// verbose("non-reopen START:");
@@ -2236,6 +2250,7 @@ public final class SolrCore implements S
lst.add("coreName", name==null ? "(null)" : name);
lst.add("startTime", new Date(startTime));
lst.add("refCount", getOpenCount());
+ lst.add("instanceDir", resourceLoader.getInstanceDir());
lst.add("indexDir", getIndexDir());
CoreDescriptor cd = getCoreDescriptor();
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Mon Oct 21 18:58:24 2013
@@ -55,6 +55,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
+import java.net.URI;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.CharacterCodingException;
@@ -250,7 +251,7 @@ public class SolrResourceLoader implemen
}
public String getConfigDir() {
- return instanceDir + "conf/";
+ return instanceDir + "conf" + File.separator;
}
public String getDataDir() {
@@ -299,27 +300,46 @@ public class SolrResourceLoader implemen
public InputStream openResource(String resource) throws IOException {
InputStream is=null;
try {
- File f0 = new File(resource);
- File f = f0;
+ File f0 = new File(resource), f = f0;
if (!f.isAbsolute()) {
// try $CWD/$configDir/$resource
- f = new File(getConfigDir() + resource);
+ f = new File(getConfigDir() + resource).getAbsoluteFile();
}
- if (f.isFile() && f.canRead()) {
+ boolean found = f.isFile() && f.canRead();
+ if (!found) { // no success with $CWD/$configDir/$resource
+ f = f0.getAbsoluteFile();
+ found = f.isFile() && f.canRead();
+ }
+ // check that we don't escape instance dir
+ if (found) {
+ if (!Boolean.parseBoolean(System.getProperty("solr.allow.unsafe.resourceloading", "false"))) {
+ final URI instanceURI = new File(getInstanceDir()).getAbsoluteFile().toURI().normalize();
+ final URI fileURI = f.toURI().normalize();
+ if (instanceURI.relativize(fileURI) == fileURI) {
+ // no URI relativize possible, so they don't share same base folder
+ throw new IOException("For security reasons, SolrResourceLoader cannot load files from outside the instance's directory: " + f +
+ "; if you want to override this safety feature and you are sure about the consequences, you can pass the system property "+
+ "-Dsolr.allow.unsafe.resourceloading=true to your JVM");
+ }
+ }
+ // relativize() returned a relative, new URI, so we are fine!
return new FileInputStream(f);
- } else if (f != f0) { // no success with $CWD/$configDir/$resource
- if (f0.isFile() && f0.canRead())
- return new FileInputStream(f0);
- }
- // delegate to the class loader (looking into $INSTANCE_DIR/lib jars)
- is = classLoader.getResourceAsStream(resource);
- if (is == null)
- is = classLoader.getResourceAsStream(getConfigDir() + resource);
+ }
+ // Delegate to the class loader (looking into $INSTANCE_DIR/lib jars).
+ // We need a ClassLoader-compatible (forward-slashes) path here!
+ is = classLoader.getResourceAsStream(resource.replace(File.separatorChar, '/'));
+ // This is a hack just for tests (it is not done in ZKResourceLoader)!
+ // -> the getConfigDir's path must not be absolute!
+ if (is == null && System.getProperty("jetty.testMode") != null && !new File(getConfigDir()).isAbsolute()) {
+ is = classLoader.getResourceAsStream((getConfigDir() + resource).replace(File.separatorChar, '/'));
+ }
+ } catch (IOException ioe) {
+ throw ioe;
} catch (Exception e) {
throw new IOException("Error opening " + resource, e);
}
if (is==null) {
- throw new IOException("Can't find resource '" + resource + "' in classpath or '" + getConfigDir() + "', cwd="+System.getProperty("user.dir"));
+ throw new IOException("Can't find resource '" + resource + "' in classpath or '" + new File(getConfigDir()).getAbsolutePath() + "'");
}
return is;
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java Mon Oct 21 18:58:24 2013
@@ -19,6 +19,8 @@ package org.apache.solr.core;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
+
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,9 +30,13 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -41,7 +47,6 @@ public class SolrXMLCoresLocator impleme
private static final Logger logger = LoggerFactory.getLogger(SolrXMLCoresLocator.class);
- private final File file;
private final String solrXmlTemplate;
private final ConfigSolrXmlOld cfg;
@@ -50,13 +55,11 @@ public class SolrXMLCoresLocator impleme
/**
* Create a new SolrXMLCoresLocator
- * @param file a File object representing the file to write out to
* @param originalXML the original content of the solr.xml file
* @param cfg the CoreContainer's config object
*/
- public SolrXMLCoresLocator(File file, String originalXML, ConfigSolrXmlOld cfg) {
+ public SolrXMLCoresLocator(String originalXML, ConfigSolrXmlOld cfg) {
this.solrXmlTemplate = buildTemplate(originalXML);
- this.file = file;
this.cfg = cfg;
}
@@ -142,19 +145,31 @@ public class SolrXMLCoresLocator impleme
}
@Override
- public final void persist(CoreContainer cc, CoreDescriptor... coreDescriptors) {
- doPersist(buildSolrXML(cc.getCoreDescriptors()));
+ public synchronized final void persist(CoreContainer cc, CoreDescriptor... coreDescriptors) {
+ List<CoreDescriptor> cds = new ArrayList<CoreDescriptor>(cc.getCoreDescriptors().size() + coreDescriptors.length);
+
+ cds.addAll(cc.getCoreDescriptors());
+ cds.addAll(Arrays.asList(coreDescriptors));
+
+ doPersist(buildSolrXML(cds));
}
protected void doPersist(String xml) {
+ File file = new File(cfg.config.getResourceLoader().getInstanceDir(), ConfigSolr.SOLR_XML_FILE);
+ Writer writer = null;
+ FileOutputStream fos = null;
try {
- Writer writer = new OutputStreamWriter(new FileOutputStream(file), Charsets.UTF_8);
+ fos = new FileOutputStream(file);
+ writer = new OutputStreamWriter(fos, Charsets.UTF_8);
writer.write(xml);
writer.close();
logger.info("Persisted core descriptions to {}", file.getAbsolutePath());
- }
- catch (IOException e) {
- logger.error("Couldn't persist core descriptions to {} : {}", file.getAbsolutePath(), e);
+ } catch (IOException e) {
+ logger.error("Couldn't persist core descriptions to {} : {}",
+ file.getAbsolutePath(), e);
+ } finally {
+ IOUtils.closeQuietly(writer);
+ IOUtils.closeQuietly(fos);
}
}
@@ -165,12 +180,14 @@ public class SolrXMLCoresLocator impleme
@Override
public void delete(CoreContainer cc, CoreDescriptor... coreDescriptors) {
- this.persist(cc, coreDescriptors);
+ // coreDescriptors is kind of a useless param - we persist the current state off cc
+ this.persist(cc);
}
@Override
public void rename(CoreContainer cc, CoreDescriptor oldCD, CoreDescriptor newCD) {
- this.persist(cc, oldCD, newCD);
+ // we don't need those params, we just write out the current cc state
+ this.persist(cc);
}
@Override
@@ -204,8 +221,8 @@ public class SolrXMLCoresLocator impleme
public static class NonPersistingLocator extends SolrXMLCoresLocator {
- public NonPersistingLocator(File file, String originalXML, ConfigSolrXmlOld cfg) {
- super(file, originalXML, cfg);
+ public NonPersistingLocator(String originalXML, ConfigSolrXmlOld cfg) {
+ super(originalXML, cfg);
this.xml = originalXML;
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java Mon Oct 21 18:58:24 2013
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.IOException;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
/**
@@ -26,6 +27,7 @@ import org.apache.lucene.store.Directory
* {@link DirectoryReader}.
*
* @see DirectoryReader#open(Directory)
+ * @see DirectoryReader#open(IndexWriter, boolean)
*/
public class StandardIndexReaderFactory extends IndexReaderFactory {
@@ -33,4 +35,9 @@ public class StandardIndexReaderFactory
public DirectoryReader newReader(Directory indexDir, SolrCore core) throws IOException {
return DirectoryReader.open(indexDir);
}
+
+ @Override
+ public DirectoryReader newReader(IndexWriter writer, SolrCore core) throws IOException {
+ return DirectoryReader.open(writer, true);
+ }
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ZkContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ZkContainer.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ZkContainer.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/core/ZkContainer.java Mon Oct 21 18:58:24 2013
@@ -227,7 +227,7 @@ public class ZkContainer {
"Could not find config name for collection:" + collection);
}
solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName,
- loader.getClassLoader(), dcore.getCoreProperties(), zkController);
+ loader.getClassLoader(), dcore.getSubstitutableProperties(), zkController);
config = getSolrConfigFromZk(zkConfigName, dcore.getConfigName(),
solrLoader);
schema = IndexSchemaFactory.buildIndexSchema(dcore.getSchemaName(),
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java Mon Oct 21 18:58:24 2013
@@ -30,6 +30,7 @@ import org.apache.lucene.util.AttributeS
import org.apache.lucene.util.AttributeReflector;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -84,15 +85,13 @@ public abstract class AnalysisRequestHan
if (!TokenizerChain.class.isInstance(analyzer)) {
- TokenStream tokenStream = null;
- try {
- tokenStream = analyzer.tokenStream(context.getFieldName(), value);
+ try (TokenStream tokenStream = analyzer.tokenStream(context.getFieldName(), value)) {
+ NamedList<List<NamedList>> namedList = new NamedList<List<NamedList>>();
+ namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(analyzeTokenStream(tokenStream), context));
+ return namedList;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
- NamedList<List<NamedList>> namedList = new NamedList<List<NamedList>>();
- namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(analyzeTokenStream(tokenStream), context));
- return namedList;
}
TokenizerChain tokenizerChain = (TokenizerChain) analyzer;
@@ -138,9 +137,8 @@ public abstract class AnalysisRequestHan
* @param analyzer The analyzer to use.
*/
protected Set<BytesRef> getQueryTokenSet(String query, Analyzer analyzer) {
- try {
+ try (TokenStream tokenStream = analyzer.tokenStream("", query)){
final Set<BytesRef> tokens = new HashSet<BytesRef>();
- final TokenStream tokenStream = analyzer.tokenStream("", query);
final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);
final BytesRef bytes = bytesAtt.getBytesRef();
@@ -152,7 +150,6 @@ public abstract class AnalysisRequestHan
}
tokenStream.end();
- tokenStream.close();
return tokens;
} catch (IOException ioe) {
throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
@@ -181,8 +178,11 @@ public abstract class AnalysisRequestHan
trackerAtt.setActPosition(position);
tokens.add(tokenStream.cloneAttributes());
}
+ tokenStream.end();
} catch (IOException ioe) {
throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
+ } finally {
+ IOUtils.closeWhileHandlingException(tokenStream);
}
return tokens;
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Mon Oct 21 18:58:24 2013
@@ -49,7 +49,10 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+
import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
+
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@@ -1002,7 +1005,7 @@ public class ReplicationHandler extends
@Override
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
- return "application/octet-stream";
+ return BinaryResponseParser.BINARY_CONTENT_TYPE;
}
@Override
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Mon Oct 21 18:58:24 2013
@@ -75,12 +75,15 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+
import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
+
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -247,13 +250,18 @@ public class SnapPuller {
params.set(CommonParams.QT, "/replication");
QueryRequest req = new QueryRequest(params);
HttpSolrServer server = new HttpSolrServer(masterUrl, myHttpClient); //XXX modify to use shardhandler
- server.setSoTimeout(60000);
- server.setConnectionTimeout(15000);
+ NamedList rsp;
try {
- return server.request(req);
+ server.setSoTimeout(60000);
+ server.setConnectionTimeout(15000);
+
+ rsp = server.request(req);
} catch (SolrServerException e) {
- throw new IOException(e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e.getMessage());
+ } finally {
+ server.shutdown();
}
+ return rsp;
}
/**
@@ -267,10 +275,9 @@ public class SnapPuller {
params.set(CommonParams.QT, "/replication");
QueryRequest req = new QueryRequest(params);
HttpSolrServer server = new HttpSolrServer(masterUrl, myHttpClient); //XXX modify to use shardhandler
- server.setSoTimeout(60000);
- server.setConnectionTimeout(15000);
-
try {
+ server.setSoTimeout(60000);
+ server.setConnectionTimeout(15000);
NamedList response = server.request(req);
List<Map<String, Object>> files = (List<Map<String,Object>>) response.get(CMD_GET_FILE_LIST);
@@ -287,6 +294,8 @@ public class SnapPuller {
} catch (SolrServerException e) {
throw new IOException(e);
+ } finally {
+ server.shutdown();
}
}
@@ -1273,9 +1282,7 @@ public class SnapPuller {
* Open a new stream using HttpClient
*/
FastInputStream getStream() throws IOException {
- HttpSolrServer s = new HttpSolrServer(masterUrl, myHttpClient, null); //XXX use shardhandler
- s.setSoTimeout(60000);
- s.setConnectionTimeout(15000);
+
ModifiableSolrParams params = new ModifiableSolrParams();
// //the method is command=filecontent
@@ -1307,7 +1314,11 @@ public class SnapPuller {
NamedList response;
InputStream is = null;
+
+ HttpSolrServer s = new HttpSolrServer(masterUrl, myHttpClient, null); //XXX use shardhandler
try {
+ s.setSoTimeout(60000);
+ s.setConnectionTimeout(15000);
QueryRequest req = new QueryRequest(params);
response = s.request(req);
is = (InputStream) response.get("stream");
@@ -1319,6 +1330,8 @@ public class SnapPuller {
//close stream on error
IOUtils.closeQuietly(is);
throw new IOException("Could not download file '" + fileName + "'", t);
+ } finally {
+ s.shutdown();
}
}
}
@@ -1534,9 +1547,7 @@ public class SnapPuller {
* Open a new stream using HttpClient
*/
FastInputStream getStream() throws IOException {
- HttpSolrServer s = new HttpSolrServer(masterUrl, myHttpClient, null); //XXX use shardhandler
- s.setSoTimeout(60000);
- s.setConnectionTimeout(15000);
+
ModifiableSolrParams params = new ModifiableSolrParams();
// //the method is command=filecontent
@@ -1568,7 +1579,10 @@ public class SnapPuller {
NamedList response;
InputStream is = null;
+ HttpSolrServer s = new HttpSolrServer(masterUrl, myHttpClient, null); //XXX use shardhandler
try {
+ s.setSoTimeout(60000);
+ s.setConnectionTimeout(15000);
QueryRequest req = new QueryRequest(params);
response = s.request(req);
is = (InputStream) response.get("stream");
@@ -1580,6 +1594,8 @@ public class SnapPuller {
//close stream on error
IOUtils.closeQuietly(is);
throw new IOException("Could not download file '" + fileName + "'", t);
+ } finally {
+ s.shutdown();
}
}
}
@@ -1590,10 +1606,16 @@ public class SnapPuller {
params.set("slave", false);
params.set(CommonParams.QT, "/replication");
HttpSolrServer server = new HttpSolrServer(masterUrl, myHttpClient); //XXX use shardhandler
- server.setSoTimeout(60000);
- server.setConnectionTimeout(15000);
- QueryRequest request = new QueryRequest(params);
- return server.request(request);
+ NamedList rsp;
+ try {
+ server.setSoTimeout(60000);
+ server.setConnectionTimeout(15000);
+ QueryRequest request = new QueryRequest(params);
+ rsp = server.request(request);
+ } finally {
+ server.shutdown();
+ }
+ return rsp;
}
static Integer readInterval(String interval) {
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Mon Oct 21 18:58:24 2013
@@ -47,7 +47,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
@@ -59,7 +61,7 @@ import static org.apache.solr.cloud.Over
import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
-import static org.apache.solr.common.cloud.DocRouter.ROUTE_FIELD;
+import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
@@ -109,6 +111,12 @@ public class CollectionsHandler extends
"Core container instance missing");
}
+ // Make sure that the core is ZKAware
+ if(!cores.isZooKeeperAware()) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Solr instance is not running in SolrCloud mode.");
+ }
+
// Pick the action
SolrParams params = req.getParams();
CollectionAction action = null;
@@ -224,13 +232,17 @@ public class CollectionsHandler extends
ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(leaderProps);
HttpSolrServer server = new HttpSolrServer(nodeProps.getBaseUrl());
- server.setConnectionTimeout(15000);
- server.setSoTimeout(30000);
- RequestSyncShard reqSyncShard = new CoreAdminRequest.RequestSyncShard();
- reqSyncShard.setCollection(collection);
- reqSyncShard.setShard(shard);
- reqSyncShard.setCoreName(nodeProps.getCoreName());
- server.request(reqSyncShard);
+ try {
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(60000);
+ RequestSyncShard reqSyncShard = new CoreAdminRequest.RequestSyncShard();
+ reqSyncShard.setCollection(collection);
+ reqSyncShard.setShard(shard);
+ reqSyncShard.setCoreName(nodeProps.getCoreName());
+ server.request(reqSyncShard);
+ } finally {
+ server.shutdown();
+ }
}
private void handleCreateAliasAction(SolrQueryRequest req,
@@ -254,7 +266,7 @@ public class CollectionsHandler extends
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
OverseerCollectionProcessor.DELETEALIAS, "name", name);
- handleResponse(OverseerCollectionProcessor.CREATEALIAS, m, rsp);
+ handleResponse(OverseerCollectionProcessor.DELETEALIAS, m, rsp);
}
private void handleDeleteAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
@@ -294,9 +306,8 @@ public class CollectionsHandler extends
NUM_SLICES,
MAX_SHARDS_PER_NODE,
CREATE_NODE_SET ,
- ROUTER,
SHARDS_PROP,
- ROUTE_FIELD);
+ "router.");
ZkNodeProps m = new ZkNodeProps(props);
@@ -307,33 +318,50 @@ public class CollectionsHandler extends
log.info("Create shard: " + req.getParamString());
req.getParams().required().check(COLLECTION_PROP, SHARD_ID_PROP);
ClusterState clusterState = coreContainer.getZkController().getClusterState();
- if(!ImplicitDocRouter.NAME.equals( clusterState.getCollection(req.getParams().get(COLLECTION_PROP)).getStr(ROUTER)))
+ if(!ImplicitDocRouter.NAME.equals( ((Map) clusterState.getCollection(req.getParams().get(COLLECTION_PROP)).get(ROUTER)).get("name") ) )
throw new SolrException(ErrorCode.BAD_REQUEST, "shards can be added only to 'implicit' collections" );
- Map<String, Object> map = OverseerCollectionProcessor.asMap(QUEUE_OPERATION, CREATESHARD);
- copyIfNotNull(req.getParams(),map,COLLECTION_PROP, SHARD_ID_PROP, REPLICATION_FACTOR);
+ Map<String, Object> map = makeMap(QUEUE_OPERATION, CREATESHARD);
+ copyIfNotNull(req.getParams(),map,COLLECTION_PROP, SHARD_ID_PROP, REPLICATION_FACTOR,CREATE_NODE_SET);
ZkNodeProps m = new ZkNodeProps(map);
handleResponse(CREATESHARD, m, rsp);
}
private static void copyIfNotNull(SolrParams params, Map<String, Object> props, String... keys) {
+ ArrayList<String> prefixes = new ArrayList<String>(1);
if(keys !=null){
for (String key : keys) {
+ if(key.endsWith(".")) {
+ prefixes.add(key);
+ continue;
+ }
String v = params.get(key);
if(v != null) props.put(key,v);
}
}
+ if(prefixes.isEmpty()) return;
+ Iterator<String> it = params.getParameterNamesIterator();
+ String prefix = null;
+ for(;it.hasNext();){
+ String name = it.next();
+ for (int i = 0; i < prefixes.size(); i++) {
+ if(name.startsWith(prefixes.get(i))){
+ String val = params.get(name);
+ if(val !=null) props.put(name,val);
+ }
+ }
+ }
}
private void handleDeleteShardAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws InterruptedException, KeeperException {
log.info("Deleting Shard : " + req.getParamString());
- String name = req.getParams().required().get("collection");
- String shard = req.getParams().required().get("shard");
+ String name = req.getParams().required().get(ZkStateReader.COLLECTION_PROP);
+ String shard = req.getParams().required().get(ZkStateReader.SHARD_ID_PROP);
Map<String,Object> props = new HashMap<String,Object>();
- props.put("collection", name);
+ props.put(ZkStateReader.COLLECTION_PROP, name);
props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.DELETESHARD);
props.put(ZkStateReader.SHARD_ID_PROP, shard);
@@ -345,13 +373,34 @@ public class CollectionsHandler extends
log.info("Splitting shard : " + req.getParamString());
String name = req.getParams().required().get("collection");
// TODO : add support for multiple shards
- String shard = req.getParams().required().get("shard");
- // TODO : add support for shard range
+ String shard = req.getParams().get("shard");
+ String rangesStr = req.getParams().get(CoreAdminParams.RANGES);
+ String splitKey = req.getParams().get("split.key");
+
+ if (splitKey == null && shard == null) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Missing required parameter: shard");
+ }
+ if (splitKey != null && shard != null) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "Only one of 'shard' or 'split.key' should be specified");
+ }
+ if (splitKey != null && rangesStr != null) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "Only one of 'ranges' or 'split.key' should be specified");
+ }
Map<String,Object> props = new HashMap<String,Object>();
props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.SPLITSHARD);
props.put("collection", name);
- props.put(ZkStateReader.SHARD_ID_PROP, shard);
+ if (shard != null) {
+ props.put(ZkStateReader.SHARD_ID_PROP, shard);
+ }
+ if (splitKey != null) {
+ props.put("split.key", splitKey);
+ }
+ if (rangesStr != null) {
+ props.put(CoreAdminParams.RANGES, rangesStr);
+ }
ZkNodeProps m = new ZkNodeProps(props);
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Mon Oct 21 18:58:24 2013
@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.DirectoryReader;
@@ -48,6 +49,7 @@ import org.apache.solr.core.CoreDescript
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrXMLCoresLocator;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
@@ -77,6 +79,8 @@ import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Future;
+import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
+
/**
*
* @since solr 1.3
@@ -229,7 +233,23 @@ public class CoreAdminHandler extends Re
List<DocRouter.Range> ranges = null;
String[] pathsArr = params.getParams("path");
- String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f
+ String rangesStr = params.get(CoreAdminParams.RANGES); // ranges=a-b,c-d,e-f
+ if (rangesStr != null) {
+ String[] rangesArr = rangesStr.split(",");
+ if (rangesArr.length == 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least one range specified to split an index");
+ } else {
+ ranges = new ArrayList<DocRouter.Range>(rangesArr.length);
+ for (String r : rangesArr) {
+ try {
+ ranges.add(DocRouter.DEFAULT.fromString(r));
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Exception parsing hexadecimal hash range: " + r, e);
+ }
+ }
+ }
+ }
+ String splitKey = params.get("split.key");
String[] newCoreNames = params.getParams("targetCore");
String cname = params.get(CoreAdminParams.CORE, "");
@@ -248,15 +268,22 @@ public class CoreAdminHandler extends Re
int partitions = pathsArr != null ? pathsArr.length : newCoreNames.length;
DocRouter router = null;
+ String routeFieldName = null;
if (coreContainer.isZooKeeperAware()) {
ClusterState clusterState = coreContainer.getZkController().getClusterState();
String collectionName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
DocCollection collection = clusterState.getCollection(collectionName);
String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
Slice slice = clusterState.getSlice(collectionName, sliceName);
- DocRouter.Range currentRange = slice.getRange();
router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
- ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
+ if (ranges == null) {
+ DocRouter.Range currentRange = slice.getRange();
+ ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
+ }
+ Map m = (Map) collection.get(DOC_ROUTER);
+ if (m != null) {
+ routeFieldName = (String) m.get("field");
+ }
}
if (pathsArr == null) {
@@ -274,7 +301,7 @@ public class CoreAdminHandler extends Re
}
- SplitIndexCommand cmd = new SplitIndexCommand(req, paths, newCores, ranges, router);
+ SplitIndexCommand cmd = new SplitIndexCommand(req, paths, newCores, ranges, router, routeFieldName, splitKey);
core.getUpdateHandler().split(cmd);
// After the split has completed, someone (here?) should start the process of replaying the buffered updates.
@@ -399,6 +426,7 @@ public class CoreAdminHandler extends Re
.put(CoreAdminParams.CORE_NODE_NAME, CoreDescriptor.CORE_NODE_NAME)
.put(CoreAdminParams.SHARD_STATE, CloudDescriptor.SHARD_STATE)
.put(CoreAdminParams.SHARD_RANGE, CloudDescriptor.SHARD_RANGE)
+ .put(CoreAdminParams.SHARD_PARENT, CloudDescriptor.SHARD_PARENT)
.put(ZkStateReader.NUM_SHARDS_PROP, CloudDescriptor.NUM_SHARDS)
.build();
@@ -409,8 +437,10 @@ public class CoreAdminHandler extends Re
String name = checkNotEmpty(params.get(CoreAdminParams.NAME),
"Missing parameter [" + CoreAdminParams.NAME + "]");
String instancedir = params.get(CoreAdminParams.INSTANCE_DIR);
- if (StringUtils.isEmpty(instancedir))
- instancedir = container.getSolrHome() + File.separator + name;
+ if (StringUtils.isEmpty(instancedir)) {
+ instancedir = name; // will be resolved later against solr.home
+ //instancedir = container.getSolrHome() + "/" + name;
+ }
Properties coreProps = new Properties();
for (String param : paramToProp.keySet()) {
@@ -429,7 +459,7 @@ public class CoreAdminHandler extends Re
coreProps.setProperty(propName, propValue);
}
- return new CoreDescriptor(container, name, instancedir, coreProps);
+ return new CoreDescriptor(container, name, instancedir, coreProps, params);
}
private static String checkNotEmpty(String value, String message) {
@@ -459,9 +489,19 @@ public class CoreAdminHandler extends Re
if (coreContainer.getZkController() != null) {
coreContainer.preRegisterInZk(dcore);
}
+
+ // make sure we can write out the descriptor first
coreContainer.getCoresLocator().create(coreContainer, dcore);
+
SolrCore core = coreContainer.create(dcore);
+
coreContainer.register(dcore.getName(), core, false);
+
+ if (coreContainer.getCoresLocator() instanceof SolrXMLCoresLocator) {
+ // hack - in this case we persist once more because a core create race might
+ // have dropped entries.
+ coreContainer.getCoresLocator().create(coreContainer);
+ }
rsp.add("core", core.getName());
}
catch (Exception ex) {
@@ -491,8 +531,8 @@ public class CoreAdminHandler extends Re
}
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Error CREATEing SolrCore '" + dcore.getName() + "': " +
- ex.getMessage(), ex);
+ "Error CREATEing SolrCore '" + dcore.getName() + "': " +
+ ex.getMessage() + rootMsg, ex);
}
}
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Mon Oct 21 18:58:24 2013
@@ -555,6 +555,7 @@ public class LukeRequestHandler extends
indexInfo.add("numDocs", reader.numDocs());
indexInfo.add("maxDoc", reader.maxDoc());
indexInfo.add("deletedDocs", reader.maxDoc() - reader.numDocs());
+ indexInfo.add("indexHeapUsageBytes", getIndexHeapUsed(reader));
indexInfo.add("version", reader.getVersion()); // TODO? Is this different then: IndexReader.getCurrentVersion( dir )?
indexInfo.add("segmentCount", reader.leaves().size());
@@ -569,6 +570,21 @@ public class LukeRequestHandler extends
return indexInfo;
}
+ /** Returns the sum of RAM bytes used by each segment */
+ private static long getIndexHeapUsed(DirectoryReader reader) {
+ long indexHeapRamBytesUsed = 0;
+ for(AtomicReaderContext atomicReaderContext : reader.leaves()) {
+ AtomicReader atomicReader = atomicReaderContext.reader();
+ if (atomicReader instanceof SegmentReader) {
+ indexHeapRamBytesUsed += ((SegmentReader) atomicReader).ramBytesUsed();
+ } else {
+ // Not supported for any reader that is not a SegmentReader
+ return -1;
+ }
+ }
+ return indexHeapRamBytesUsed;
+ }
+
// Get terribly detailed information about a particular field. This is a very expensive call, use it with caution
// especially on large indexes!
@SuppressWarnings("unchecked")
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Mon Oct 21 18:58:24 2013
@@ -152,7 +152,11 @@ public class HttpShardHandler extends Sh
String url = urls.get(0);
srsp.setShardAddress(url);
SolrServer server = new HttpSolrServer(url, httpClient);
- ssr.nl = server.request(req);
+ try {
+ ssr.nl = server.request(req);
+ } finally {
+ server.shutdown();
+ }
} else {
LBHttpSolrServer.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls);
ssr.nl = rsp.getResponse();
Modified: lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java (original)
+++ lucene/dev/branches/lucene4956/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java Mon Oct 21 18:58:24 2013
@@ -45,7 +45,9 @@ import org.apache.solr.schema.IndexSchem
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocListAndSet;
+import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SolrReturnFields;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,12 +72,20 @@ public class MoreLikeThisComponent exten
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
+ ReturnFields returnFields = new SolrReturnFields( rb.req );
+
+ int flags = 0;
+ if (returnFields.wantsScore()) {
+ flags |= SolrIndexSearcher.GET_SCORES;
+ }
+
+ rb.setFieldFlags(flags);
+
if (params.getBool(MoreLikeThisParams.MLT, false)) {
log.debug("Starting MoreLikeThis.Process. isShard: "
+ params.getBool(ShardParams.IS_SHARD));
SolrIndexSearcher searcher = rb.req.getSearcher();
- int mltcount = params.getInt(MoreLikeThisParams.DOC_COUNT, 5);
if (params.getBool(ShardParams.IS_SHARD, false)) {
if (params.get(MoreLikeThisComponent.DIST_DOC_ID) == null) {
if (rb.getResults().docList.size() == 0) {
@@ -86,7 +96,7 @@ public class MoreLikeThisComponent exten
MoreLikeThisHandler.MoreLikeThisHelper mlt = new MoreLikeThisHandler.MoreLikeThisHelper(
params, searcher);
-
+
NamedList<BooleanQuery> bQuery = mlt.getMoreLikeTheseQuery(rb
.getResults().docList);
@@ -105,13 +115,13 @@ public class MoreLikeThisComponent exten
rb.rsp.add("moreLikeThis", temp);
} else {
NamedList<DocList> sim = getMoreLikeThese(rb, rb.req.getSearcher(),
- rb.getResults().docList, mltcount);
+ rb.getResults().docList, flags);
rb.rsp.add("moreLikeThis", sim);
}
} else {
// non distrib case
NamedList<DocList> sim = getMoreLikeThese(rb, rb.req.getSearcher(), rb.getResults().docList,
- mltcount);
+ flags);
rb.rsp.add("moreLikeThis", sim);
}
}