You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by er...@apache.org on 2017/04/03 20:29:39 UTC
lucene-solr:master: SOLR-8906: Make transient core cache pluggable
Repository: lucene-solr
Updated Branches:
refs/heads/master 2e545d78f -> 52632cfc0
SOLR-8906: Make transient core cache pluggable
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/52632cfc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/52632cfc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/52632cfc
Branch: refs/heads/master
Commit: 52632cfc0c0c945cff2e769e6c2dc4dc9a5da400
Parents: 2e545d7
Author: Erick Erickson <er...@apache.org>
Authored: Mon Apr 3 13:27:12 2017 -0700
Committer: Erick Erickson <er...@apache.org>
Committed: Mon Apr 3 13:27:12 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/core/CoreContainer.java | 25 ++-
.../java/org/apache/solr/core/NodeConfig.java | 25 ++-
.../java/org/apache/solr/core/SolrCores.java | 166 ++++++++++------
.../org/apache/solr/core/SolrXmlConfig.java | 5 +
.../solr/core/TransientSolrCoreCache.java | 127 ++++++++++++
.../core/TransientSolrCoreCacheDefault.java | 198 +++++++++++++++++++
.../core/TransientSolrCoreCacheFactory.java | 85 ++++++++
.../TransientSolrCoreCacheFactoryDefault.java | 31 +++
solr/core/src/test-files/solr/solr.xml | 5 +
.../org/apache/solr/cloud/ZkControllerTest.java | 9 +-
.../org/apache/solr/core/TestCoreDiscovery.java | 7 +-
.../org/apache/solr/core/TestLazyCores.java | 53 ++++-
.../java/org/apache/solr/SolrTestCaseJ4.java | 13 +-
14 files changed, 662 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index cd4f7f5..6fe4cc0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -188,6 +188,8 @@ Other Changes
* SOLR-9601: Redone DataImportHandler 'tika' example, removing all unused and irrelevant definitions (Alexandre Rafalovitch)
+* SOLR-8906: Make transient core cache pluggable (Erick Erickson)
+
================== 6.5.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 5ec34ba..1ef036a 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -130,6 +130,7 @@ public class CoreContainer {
protected CoreAdminHandler coreAdminHandler = null;
protected CollectionsHandler collectionsHandler = null;
+ protected TransientSolrCoreCache transientSolrCoreCache = null;
private InfoHandler infoHandler;
protected ConfigSetsHandler configSetsHandler = null;
@@ -144,6 +145,8 @@ public class CoreContainer {
private UpdateShardHandler updateShardHandler;
+ private TransientSolrCoreCacheFactory transientCoreCache;
+
private ExecutorService coreContainerWorkExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(
new DefaultSolrThreadFactory("coreContainerWorkExecutor") );
@@ -492,7 +495,7 @@ public class CoreContainer {
updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());
updateShardHandler.initializeMetrics(metricManager, SolrInfoMBean.Group.node.toString(), "updateShardHandler");
- solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
+ transientCoreCache = TransientSolrCoreCacheFactory.newInstance(loader, this);
logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
@@ -535,9 +538,9 @@ public class CoreContainer {
String registryName = SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node);
metricManager.registerGauge(registryName, () -> solrCores.getCores().size(),
true, "loaded", SolrInfoMBean.Category.CONTAINER.toString(), "cores");
- metricManager.registerGauge(registryName, () -> solrCores.getCoreNames().size() - solrCores.getCores().size(),
+ metricManager.registerGauge(registryName, () -> solrCores.getLoadedCoreNames().size() - solrCores.getCores().size(),
true, "lazy",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
- metricManager.registerGauge(registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getCoreNames().size(),
+ metricManager.registerGauge(registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getLoadedCoreNames().size(),
true, "unloaded",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(registryName, () -> cfg.getCoreRootDirectory().toFile().getTotalSpace(),
true, "totalSpace", SolrInfoMBean.Category.CONTAINER.toString(), "fs");
@@ -629,6 +632,16 @@ public class CoreContainer {
}
}
+ public TransientSolrCoreCache getTransientCacheHandler() {
+
+ if (transientCoreCache == null) {
+ log.error("No transient handler has been defined. Check solr.xml to see if an attempt to provide a custom " +
+ "TransientSolrCoreCacheFactory was done incorrectly since the default should have been used otherwise.");
+ return null;
+ }
+ return transientCoreCache.getTransientSolrCoreCache();
+ }
+
public void securityNodeChanged() {
log.info("Security node changed, reloading security.json");
reloadSecurityProperties();
@@ -1076,10 +1089,10 @@ public class CoreContainer {
}
/**
- * @return a Collection of the names that cores are mapped to
+ * @return a Collection of the names that loaded cores are mapped to
*/
public Collection<String> getCoreNames() {
- return solrCores.getCoreNames();
+ return solrCores.getLoadedCoreNames();
}
/** This method is currently experimental.
@@ -1092,6 +1105,8 @@ public class CoreContainer {
/**
* get a list of all the cores that are currently loaded
* @return a list of al lthe available core names in either permanent or transient core lists.
+ *
+ * Note: this implies that the core is loaded
*/
public Collection<String> getAllCoreNames() {
return solrCores.getAllCoreNames();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/NodeConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
index 258fd14..de2dcea 100644
--- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
@@ -52,6 +52,8 @@ public class NodeConfig {
private final Integer coreLoadThreads;
+ @Deprecated
+ // This should be part of the transientCacheConfig, remove in 7.0
private final int transientCacheSize;
private final boolean useSchemaCache;
@@ -62,6 +64,8 @@ public class NodeConfig {
private final PluginInfo[] metricReporterPlugins;
+ private final PluginInfo transientCacheConfig;
+
private NodeConfig(String nodeName, Path coreRootDirectory, Path configSetBaseDirectory, String sharedLibDirectory,
PluginInfo shardHandlerFactoryConfig, UpdateShardHandlerConfig updateShardHandlerConfig,
String coreAdminHandlerClass, String collectionsAdminHandlerClass,
@@ -69,7 +73,7 @@ public class NodeConfig {
LogWatcherConfig logWatcherConfig, CloudConfig cloudConfig, Integer coreLoadThreads,
int transientCacheSize, boolean useSchemaCache, String managementPath, SolrResourceLoader loader,
Properties solrProperties, PluginInfo[] backupRepositoryPlugins,
- PluginInfo[] metricReporterPlugins) {
+ PluginInfo[] metricReporterPlugins, PluginInfo transientCacheConfig) {
this.nodeName = nodeName;
this.coreRootDirectory = coreRootDirectory;
this.configSetBaseDirectory = configSetBaseDirectory;
@@ -90,6 +94,7 @@ public class NodeConfig {
this.solrProperties = solrProperties;
this.backupRepositoryPlugins = backupRepositoryPlugins;
this.metricReporterPlugins = metricReporterPlugins;
+ this.transientCacheConfig = transientCacheConfig;
if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@@ -182,6 +187,8 @@ public class NodeConfig {
return metricReporterPlugins;
}
+ public PluginInfo getTransientCachePluginInfo() { return transientCacheConfig; }
+
public static class NodeConfigBuilder {
private Path coreRootDirectory;
@@ -195,13 +202,16 @@ public class NodeConfig {
private String configSetsHandlerClass = DEFAULT_CONFIGSETSHANDLERCLASS;
private LogWatcherConfig logWatcherConfig = new LogWatcherConfig(true, null, null, 50);
private CloudConfig cloudConfig;
- private Integer coreLoadThreads;
+ private int coreLoadThreads = DEFAULT_CORE_LOAD_THREADS;
+ @Deprecated
+ //Remove in 7.0 and put it all in the transientCache element in solrconfig.xml
private int transientCacheSize = DEFAULT_TRANSIENT_CACHE_SIZE;
private boolean useSchemaCache = false;
private String managementPath;
private Properties solrProperties = new Properties();
private PluginInfo[] backupRepositoryPlugins;
private PluginInfo[] metricReporterPlugins;
+ private PluginInfo transientCacheConfig;
private final SolrResourceLoader loader;
private final String nodeName;
@@ -210,7 +220,7 @@ public class NodeConfig {
//No:of core load threads in cloud mode is set to a default of 8
public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 8;
- private static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE;
+ public static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE;
private static final String DEFAULT_ADMINHANDLERCLASS = "org.apache.solr.handler.admin.CoreAdminHandler";
private static final String DEFAULT_INFOHANDLERCLASS = "org.apache.solr.handler.admin.InfoHandler";
@@ -284,6 +294,8 @@ public class NodeConfig {
return this;
}
+ // Remove in Solr 7.0
+ @Deprecated
public NodeConfigBuilder setTransientCacheSize(int transientCacheSize) {
this.transientCacheSize = transientCacheSize;
return this;
@@ -313,12 +325,17 @@ public class NodeConfig {
this.metricReporterPlugins = metricReporterPlugins;
return this;
}
+
+ public NodeConfigBuilder setSolrCoreCacheFactoryConfig(PluginInfo transientCacheConfig) {
+ this.transientCacheConfig = transientCacheConfig;
+ return this;
+ }
public NodeConfig build() {
return new NodeConfig(nodeName, coreRootDirectory, configSetBaseDirectory, sharedLibDirectory, shardHandlerFactoryConfig,
updateShardHandlerConfig, coreAdminHandlerClass, collectionsAdminHandlerClass, infoHandlerClass, configSetsHandlerClass,
logWatcherConfig, cloudConfig, coreLoadThreads, transientCacheSize, useSchemaCache, managementPath, loader, solrProperties,
- backupRepositoryPlugins, metricReporterPlugins);
+ backupRepositoryPlugins, metricReporterPlugins, transientCacheConfig);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/SolrCores.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCores.java b/solr/core/src/java/org/apache/solr/core/SolrCores.java
index b25e9bb..40d5115 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCores.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCores.java
@@ -17,6 +17,7 @@
package org.apache.solr.core;
import com.google.common.collect.Lists;
+import org.apache.http.annotation.Experimental;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.logging.MDCLoggingContext;
@@ -32,6 +33,8 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Observable;
+import java.util.Observer;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
@@ -39,15 +42,12 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
-class SolrCores {
+class SolrCores implements Observer {
private static Object modifyLock = new Object(); // for locking around manipulating any of the core maps.
private final Map<String, SolrCore> cores = new LinkedHashMap<>(); // For "permanent" cores
- //WARNING! The _only_ place you put anything into the list of transient cores is with the putTransientCore method!
- private Map<String, SolrCore> transientCores = new LinkedHashMap<>(); // For "lazily loaded" cores
-
- private final Map<String, CoreDescriptor> dynamicDescriptors = new LinkedHashMap<>();
+ private final Map<String, CoreDescriptor> lazyDescriptors = new LinkedHashMap<>();
private final CoreContainer container;
@@ -66,33 +66,19 @@ class SolrCores {
SolrCores(CoreContainer container) {
this.container = container;
}
-
- // Trivial helper method for load, note it implements LRU on transient cores. Also note, if
- // there is no setting for max size, nothing is done and all cores go in the regular "cores" list
- protected void allocateLazyCores(final int cacheSize, final SolrResourceLoader loader) {
- if (cacheSize != Integer.MAX_VALUE) {
- log.info("Allocating transient cache for {} transient cores", cacheSize);
- transientCores = new LinkedHashMap<String, SolrCore>(cacheSize, 0.75f, true) {
- @Override
- protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
- if (size() > cacheSize) {
- synchronized (modifyLock) {
- SolrCore coreToClose = eldest.getValue();
- log.info("Closing transient core [{}]", coreToClose.getName());
- pendingCloses.add(coreToClose); // Essentially just queue this core up for closing.
- modifyLock.notifyAll(); // Wakes up closer thread too
- }
- return true;
- }
- return false;
- }
- };
- }
- }
-
- protected void putDynamicDescriptor(String rawName, CoreDescriptor p) {
+
+ protected void putDynamicDescriptor(String rawName, CoreDescriptor cd) {
synchronized (modifyLock) {
- dynamicDescriptors.put(rawName, p);
+ if (cd.isTransient()) {
+ if (container.getTransientCacheHandler() != null) {
+ container.getTransientCacheHandler().addTransientDescriptor(rawName, cd);
+ } else {
+ log.error("Tried to add transient core to transient handler, but no transient core handler has been found. "
+ + " Descriptor: " + cd.toString());
+ }
+ } else {
+ lazyDescriptors.put(rawName, cd);
+ }
}
}
@@ -101,19 +87,25 @@ class SolrCores {
protected void close() {
Collection<SolrCore> coreList = new ArrayList<>();
+
+ TransientSolrCoreCache transientSolrCoreCache = container.getTransientCacheHandler();
+ // Release observer
+ if (transientSolrCoreCache != null) {
+ transientSolrCoreCache.close();
+ }
+
// It might be possible for one of the cores to move from one list to another while we're closing them. So
// loop through the lists until they're all empty. In particular, the core could have moved from the transient
// list to the pendingCloses list.
-
do {
coreList.clear();
synchronized (modifyLock) {
// make a copy of the cores then clear the map so the core isn't handed out to a request again
coreList.addAll(cores.values());
cores.clear();
-
- coreList.addAll(transientCores.values());
- transientCores.clear();
+ if (transientSolrCoreCache != null) {
+ coreList.addAll(transientSolrCoreCache.prepareForShutdown());
+ }
coreList.addAll(pendingCloses);
pendingCloses.clear();
@@ -147,10 +139,12 @@ class SolrCores {
//WARNING! This should be the _only_ place you put anything into the list of transient cores!
protected SolrCore putTransientCore(NodeConfig cfg, String name, SolrCore core, SolrResourceLoader loader) {
- SolrCore retCore;
+ SolrCore retCore = null;
log.info("Opening transient core {}", name);
synchronized (modifyLock) {
- retCore = transientCores.put(name, core);
+ if (container.getTransientCacheHandler() != null) {
+ retCore = container.getTransientCacheHandler().addCore(name, core);
+ }
}
return retCore;
}
@@ -161,6 +155,17 @@ class SolrCores {
}
}
+ /**
+ *
+ * @return A list of "permanent" cores, i.e. cores that may not be swapped out and are currently loaded.
+ *
+ * A core may be non-transient but still lazily loaded. If it is "permanent" and lazy-load _and_
+ * not yet loaded it will _not_ be returned by this call.
+ *
+ * Note: This is one of the places where SolrCloud is incompatible with Transient Cores. This call is used in
+ * cancelRecoveries, transient cores don't participate.
+ */
+
List<SolrCore> getCores() {
List<SolrCore> lst = new ArrayList<>();
@@ -170,16 +175,34 @@ class SolrCores {
}
}
- Set<String> getCoreNames() {
+ /**
+ * Gets the cores that are currently loaded, i.e. cores that have
+ * 1> loadOnStartup=true and are either not-transient or, if transient, have been loaded and have not been swapped out
+ * 2> loadOnStartup=false and have been loaded but either non-transient or have not been swapped out.
+ *
+ * Put another way, this will not return any names of cores that are lazily loaded but have not been called for yet
+ * or are transient and either not loaded or have been swapped out.
+ *
+ * @return List of currently loaded cores.
+ */
+ Set<String> getLoadedCoreNames() {
Set<String> set = new TreeSet<>();
synchronized (modifyLock) {
set.addAll(cores.keySet());
- set.addAll(transientCores.keySet());
+ if (container.getTransientCacheHandler() != null) {
+ set.addAll(container.getTransientCacheHandler().getLoadedCoreNames());
+ }
}
return set;
}
+ /** This method is currently experimental.
+ * @return a Collection of the names that a specific core is mapped to.
+ *
+ * Note: this implies that the core is loaded
+ */
+ @Experimental
List<String> getCoreNames(SolrCore core) {
List<String> lst = new ArrayList<>();
@@ -189,26 +212,26 @@ class SolrCores {
lst.add(entry.getKey());
}
}
- for (Map.Entry<String, SolrCore> entry : transientCores.entrySet()) {
- if (core == entry.getValue()) {
- lst.add(entry.getKey());
- }
+ if (container.getTransientCacheHandler() != null) {
+ lst.addAll(container.getTransientCacheHandler().getNamesForCore(core));
}
}
return lst;
}
/**
- * Gets a list of all cores, loaded and unloaded (dynamic)
+ * Gets a list of all cores, loaded and unloaded
*
- * @return all cores names, whether loaded or unloaded.
+ * @return all cores names, whether loaded or unloaded, transient or permenent.
*/
public Collection<String> getAllCoreNames() {
Set<String> set = new TreeSet<>();
synchronized (modifyLock) {
set.addAll(cores.keySet());
- set.addAll(transientCores.keySet());
- set.addAll(dynamicDescriptors.keySet());
+ if (container.getTransientCacheHandler() != null) {
+ set.addAll(container.getTransientCacheHandler().getAllCoreNames());
+ }
+ set.addAll(lazyDescriptors.keySet());
}
return set;
}
@@ -251,14 +274,15 @@ class SolrCores {
protected SolrCore remove(String name) {
synchronized (modifyLock) {
- SolrCore tmp = cores.remove(name);
- SolrCore ret = null;
- ret = (ret == null) ? tmp : ret;
+ SolrCore ret = cores.remove(name);
// It could have been a newly-created core. It could have been a transient core. The newly-created cores
// in particular should be checked. It could have been a dynamic core.
- tmp = transientCores.remove(name);
- ret = (ret == null) ? tmp : ret;
- dynamicDescriptors.remove(name);
+ TransientSolrCoreCache transientHandler = container.getTransientCacheHandler();
+ if (ret == null && transientHandler != null) {
+ ret = transientHandler.removeCore(name);
+ transientHandler.removeTransientDescriptor(name);
+ }
+ lazyDescriptors.remove(name);
return ret;
}
}
@@ -268,8 +292,8 @@ class SolrCores {
synchronized (modifyLock) {
SolrCore core = cores.get(name);
- if (core == null) {
- core = transientCores.get(name);
+ if (core == null && container.getTransientCacheHandler() != null) {
+ core = container.getTransientCacheHandler().getCore(name);
}
if (core != null && incRefCount) {
@@ -282,7 +306,9 @@ class SolrCores {
protected CoreDescriptor getDynamicDescriptor(String name) {
synchronized (modifyLock) {
- return dynamicDescriptors.get(name);
+ CoreDescriptor cd = lazyDescriptors.get(name);
+ if (cd != null || container.getTransientCacheHandler() == null) return cd;
+ return container.getTransientCacheHandler().getTransientDescriptor(name);
}
}
@@ -295,7 +321,7 @@ class SolrCores {
if (cores.containsKey(name)) {
return true;
}
- if (transientCores.containsKey(name)) {
+ if (container.getTransientCacheHandler() != null && container.getTransientCacheHandler().containsCore(name)) {
// Check pending
for (SolrCore core : pendingCloses) {
if (core.getName().equals(name)) {
@@ -314,7 +340,7 @@ class SolrCores {
if (cores.containsKey(name)) {
return true;
}
- if (transientCores.containsKey(name)) {
+ if (container.getTransientCacheHandler() != null && container.getTransientCacheHandler().containsCore(name)) {
return true;
}
}
@@ -324,13 +350,16 @@ class SolrCores {
protected CoreDescriptor getUnloadedCoreDescriptor(String cname) {
synchronized (modifyLock) {
- CoreDescriptor desc = dynamicDescriptors.get(cname);
+ CoreDescriptor desc = lazyDescriptors.get(cname);
if (desc == null) {
- return null;
+ if (container.getTransientCacheHandler() == null) return null;
+ desc = container.getTransientCacheHandler().getTransientDescriptor(cname);
+ if (desc == null) {
+ return null;
+ }
}
return new CoreDescriptor(cname, desc);
}
-
}
// Wait here until any pending operations (load, unload or reload) are completed on this core.
@@ -412,9 +441,9 @@ class SolrCores {
synchronized (modifyLock) {
if (cores.containsKey(coreName))
return cores.get(coreName).getCoreDescriptor();
- if (dynamicDescriptors.containsKey(coreName))
- return dynamicDescriptors.get(coreName);
- return null;
+ if (lazyDescriptors.containsKey(coreName) || container.getTransientCacheHandler() == null)
+ return lazyDescriptors.get(coreName);
+ return container.getTransientCacheHandler().getTransientDescriptor(coreName);
}
}
@@ -494,4 +523,13 @@ class SolrCores {
}
return false;
}
+
+ // Let transient cache implementation tell us when it ages out a corel
+ @Override
+ public void update(Observable o, Object arg) {
+ synchronized (modifyLock) {
+ pendingCloses.add((SolrCore) arg); // Essentially just queue this core up for closing.
+ modifyLock.notifyAll(); // Wakes up closer thread too
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
index 951d8d5..b37bd52 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -91,6 +91,7 @@ public class SolrXmlConfig {
NodeConfig.NodeConfigBuilder configBuilder = new NodeConfig.NodeConfigBuilder(nodeName, config.getResourceLoader());
configBuilder.setUpdateShardHandlerConfig(updateConfig);
configBuilder.setShardHandlerFactoryConfig(getShardHandlerFactoryPluginInfo(config));
+ configBuilder.setSolrCoreCacheFactoryConfig(getTransientCoreCacheFactoryPluginInfo(config));
configBuilder.setLogWatcherConfig(loadLogWatcherConfig(config, "solr/logging/*[@name]", "solr/logging/watcher/*[@name]"));
configBuilder.setSolrProperties(loadProperties(config));
if (cloudConfig != null)
@@ -456,5 +457,9 @@ public class SolrXmlConfig {
}
return configs;
}
+ private static PluginInfo getTransientCoreCacheFactoryPluginInfo(Config config) {
+ Node node = config.getNode("solr/transientCoreCacheFactory", false);
+ return (node == null) ? null : new PluginInfo(node, "transientCoreCacheFactory", false, true);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCache.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCache.java b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCache.java
new file mode 100644
index 0000000..63df02b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCache.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Observable;
+import java.util.Set;
+
+import org.apache.http.annotation.Experimental;
+
+/**
+ * The base class for custom transient core maintenance. Any custom plugin that want's to take control of transient
+ * caches (i.e. any core defined with transient=true) should override this class.
+ *
+ * Register your plugin in solr.xml similarly to:
+ *
+ * <transientCoreCacheFactory name="transientCoreCacheFactory" class="TransientSolrCoreCacheFactoryDefault">
+ * <int name="transientCacheSize">4</int>
+ * </transientCoreCacheFactory>
+ *
+ *
+ * WARNING: There is quite a bit of higher-level locking done by the CoreContainer to avoid various race conditions
+ * etc. You should _only_ manipulate them within the method calls designed to change them. E.g.
+ * only add to the transient core descriptors in addTransientDescriptor etc.
+ *
+ * Trust the higher-level code (mainly SolrCores and CoreContainer) to call the appropriate operations when
+ * necessary and to coordinate shutting down cores, manipulating the internal structures and the like..
+ *
+ * The only real action you should _initiate_ is to close a core for whatever reason, and do that by
+ * calling notifyObservers(coreToClose); The observer will call back to removeCore(name) at the appropriate
+ * time. There is no need to directly remove the core _at that time_ from the transientCores list, a call
+ * will come back to this class when CoreContainer is closing this core.
+ *
+ * CoreDescriptors are read-once. During "core discovery" all valid descriptors are enumerated and added to
+ * the appropriate list. Thereafter, they are NOT re-read from disk. In those situations where you want
+ * to re-define the coreDescriptor, maintain a "side list" of changed core descriptors. Then override
+ * getTransientDescriptor to return your new core descriptor. NOTE: assuming you've already closed the
+ * core, the _next_ time that core is required getTransientDescriptor will be called and if you return the
+ * new core descriptor your re-definition should be honored. You'll have to maintain this list for the
+ * duration of this Solr instance running. If you persist the coreDescriptor, then next time Solr starts
+ * up the new definition will be read.
+ *
+ *
+ * If you need to manipulate the return, for instance block a core from being loaded for some period of time, override
+ * say getTransientDescriptor and return null.
+ *
+ * In particular, DO NOT reach into the transientCores structure from a method called to manipulate core descriptors
+ * or vice-versa.
+ */
+public abstract class TransientSolrCoreCache extends Observable {
+
+ // Gets the core container that encloses this cache.
+ public abstract CoreContainer getContainer();
+
+ // Add the newly-opened core to the list of open cores.
+ public abstract SolrCore addCore(String name, SolrCore core);
+
+ // Return the names of all possible cores, whether they are currently loaded or not.
+ public abstract Set<String> getAllCoreNames();
+
+ // Return the names of all currently loaded cores
+ public abstract Set<String> getLoadedCoreNames();
+
+ // Remove a core from the internal structures, presumably it
+ // being closed. If the core is re-opened, it will be readded by CoreContainer.
+ public abstract SolrCore removeCore(String name);
+
+ // Get the core associated with the name. Return null if you don't want this core to be used.
+ public abstract SolrCore getCore(String name);
+
+ // reutrn true if the cache contains the named core.
+ public abstract boolean containsCore(String name);
+
+ // This method will be called when the container is to be shut down. It should return all
+ // transient solr cores and clear any internal structures that hold them.
+ public abstract Collection<SolrCore> prepareForShutdown();
+
+ // These methods allow the implementation to maintain control over the core descriptors.
+
+ // This method will only be called during core discovery at startup.
+ public abstract void addTransientDescriptor(String rawName, CoreDescriptor cd);
+
+ // This method is used when opening cores and the like. If you want to change a core's descriptor, override this
+ // method and return the current core descriptor.
+ public abstract CoreDescriptor getTransientDescriptor(String name);
+
+
+ // Remove the core descriptor from your list of transient descriptors.
+ public abstract CoreDescriptor removeTransientDescriptor(String name);
+
+ // Find all the names a specific core is mapped to. Should not return null, return empty set instead.
+ @Experimental
+ public List<String> getNamesForCore(SolrCore core) {
+ return Collections.emptyList();
+ }
+
+ /**
+ * Must be called in order to free resources!
+ */
+ public abstract void close();
+
+
+ // These two methods allow custom implementations to communicate arbitrary information as necessary.
+ public abstract int getStatus(String coreName);
+ public abstract void setStatus(String coreName, int status);
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheDefault.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheDefault.java b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheDefault.java
new file mode 100644
index 0000000..e1fd748
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheDefault.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Observer;
+import java.util.Set;
+
+import org.apache.solr.common.util.NamedList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TransientSolrCoreCacheDefault extends TransientSolrCoreCache {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private int cacheSize = NodeConfig.NodeConfigBuilder.DEFAULT_TRANSIENT_CACHE_SIZE;
+
+ protected Observer observer;
+ protected CoreContainer coreContainer;
+
+ protected final Map<String, CoreDescriptor> transientDescriptors = new LinkedHashMap<>();
+
+ //WARNING! The _only_ place you put anything into the list of transient cores is with the putTransientCore method!
+ protected Map<String, SolrCore> transientCores = new LinkedHashMap<>(); // For "lazily loaded" cores
+
+ /**
+ * @param container The enclosing CoreContainer. It allows us to access everything we need.
+ */
+ public TransientSolrCoreCacheDefault(final CoreContainer container) {
+ this.coreContainer = container;
+ this.observer= coreContainer.solrCores;
+
+ NodeConfig cfg = container.getNodeConfig();
+ if (cfg.getTransientCachePluginInfo() == null) {
+ // Still handle just having transientCacheSize defined in the body of solr.xml not in a transient handler clause.
+ // deprecate this for 7.0?
+ this.cacheSize = cfg.getTransientCacheSize();
+ } else {
+ NamedList args = cfg.getTransientCachePluginInfo().initArgs;
+ Object obj = args.get("transientCacheSize");
+ if (obj != null) {
+ this.cacheSize = (int) obj;
+ }
+ }
+ doInit();
+ }
+ // This just moves the
+ private void doInit() {
+ NodeConfig cfg = coreContainer.getNodeConfig();
+ if (cfg.getTransientCachePluginInfo() == null) {
+ // Still handle just having transientCacheSize defined in the body of solr.xml not in a transient handler clause.
+ this.cacheSize = cfg.getTransientCacheSize();
+ } else {
+ NamedList args = cfg.getTransientCachePluginInfo().initArgs;
+ Object obj = args.get("transientCacheSize");
+ if (obj != null) {
+ this.cacheSize = (int) obj;
+ }
+ }
+
+ log.info("Allocating transient cache for {} transient cores", cacheSize);
+ addObserver(this.observer);
+ // it's possible for cache
+ if (cacheSize < 0) { // Trap old flag
+ cacheSize = Integer.MAX_VALUE;
+ }
+ // Now don't allow ridiculous allocations here, if the size is > 1,000, we'll just deal with
+ // adding cores as they're opened. This blows up with the marker value of -1.
+ transientCores = new LinkedHashMap<String, SolrCore>(Math.min(cacheSize, 1000), 0.75f, true) {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
+ if (size() > cacheSize) {
+ SolrCore coreToClose = eldest.getValue();
+ setChanged();
+ notifyObservers(coreToClose);
+ log.info("Closing transient core [{}]", coreToClose.getName());
+ return true;
+ }
+ return false;
+ }
+ };
+ }
+
+
+ @Override
+ public Collection<SolrCore> prepareForShutdown() {
+ // Returna copy of the values
+ List<SolrCore> ret = new ArrayList(transientCores.values());
+ transientCores.clear();
+ return ret;
+ }
+
+ @Override
+ public CoreContainer getContainer() { return this.coreContainer; }
+
+ @Override
+ public SolrCore addCore(String name, SolrCore core) {
+ return transientCores.put(name, core);
+ }
+
+ @Override
+ public Set<String> getAllCoreNames() {
+ return transientDescriptors.keySet();
+ }
+
+ @Override
+ public Set<String> getLoadedCoreNames() {
+ return transientCores.keySet();
+ }
+
+ // Remove a core from the internal structures, presumably it
+ // being closed. If the core is re-opened, it will be readded by CoreContainer.
+ @Override
+ public SolrCore removeCore(String name) {
+ return transientCores.remove(name);
+ }
+
+ // Get the core associated with the name. Return null if you don't want this core to be used.
+ @Override
+ public SolrCore getCore(String name) {
+ return transientCores.get(name);
+ }
+
+ @Override
+ public boolean containsCore(String name) {
+ return transientCores.containsKey(name);
+ }
+
+ // These methods allow the implementation to maintain control over the core descriptors.
+
+
+ // This method will only be called during core discovery at startup.
+ @Override
+ public void addTransientDescriptor(String rawName, CoreDescriptor cd) {
+ transientDescriptors.put(rawName, cd);
+ }
+
+ // This method is used when opening cores and the like. If you want to change a core's descriptor, override this
+ // method and return the current core descriptor.
+ @Override
+ public CoreDescriptor getTransientDescriptor(String name) {
+ return transientDescriptors.get(name);
+ }
+
+ @Override
+ public CoreDescriptor removeTransientDescriptor(String name) {
+ return transientDescriptors.remove(name);
+ }
+
+ @Override
+ public List<String> getNamesForCore(SolrCore core) {
+ List<String> ret = new ArrayList<>();
+ for (Map.Entry<String, SolrCore> entry : transientCores.entrySet()) {
+ if (core == entry.getValue()) {
+ ret.add(entry.getKey());
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Must be called in order to free resources!
+ */
+ @Override
+ public void close() {
+ deleteObserver(this.observer);
+ }
+
+
+ // For custom implementations to communicate arbitrary information as necessary.
+ @Override
+ public int getStatus(String coreName) { return 0; } //no_op for default handler.
+
+ @Override
+ public void setStatus(String coreName, int status) {} //no_op for default handler.
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactory.java b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactory.java
new file mode 100644
index 0000000..b3b8cf0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactory.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Collections;
+import java.util.Locale;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.solr.util.plugin.PluginInfoInitialized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An interface that allows custom transient caches to be maintained with different implementations
+ */
+public abstract class TransientSolrCoreCacheFactory {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private CoreContainer coreContainer = null;
+
+ public abstract TransientSolrCoreCache getTransientSolrCoreCache();
+ /**
+ * Create a new TransientSolrCoreCacheFactory instance
+ *
+ * @param loader a SolrResourceLoader used to find the TransientSolrCacheFactory classes
+ * @param coreContainer CoreContainer that encloses all the Solr cores.
+ * @return a new, initialized TransientSolrCoreCache instance
+ */
+
+ public static TransientSolrCoreCacheFactory newInstance(SolrResourceLoader loader, CoreContainer coreContainer) {
+ PluginInfo info = coreContainer.getConfig().getTransientCachePluginInfo();
+ if (info == null) { // definition not in our solr.xml file, use default
+ info = DEFAULT_TRANSIENT_SOLR_CACHE_INFO;
+ }
+
+ try {
+ // According to the docs, this returns a TransientSolrCoreCacheFactory with the default c'tor
+ TransientSolrCoreCacheFactory tccf = loader.findClass(info.className, TransientSolrCoreCacheFactory.class).newInstance();
+
+ // OK, now we call it's init method.
+ if (PluginInfoInitialized.class.isAssignableFrom(tccf.getClass()))
+ PluginInfoInitialized.class.cast(tccf).init(info);
+ tccf.setCoreContainer(coreContainer);
+ return tccf;
+ } catch (Exception e) {
+ // Many things could cuse this, bad solrconfig, mis-typed class name, whatever. However, this should not
+ // keep the enclosing coreContainer from instantiating, so log an error and continue.
+ log.error(String.format(Locale.ROOT, "Error instantiating TransientSolrCoreCacheFactory class [%s]: %s",
+ info.className, e.getMessage()));
+ return null;
+ }
+
+ }
+ public static final PluginInfo DEFAULT_TRANSIENT_SOLR_CACHE_INFO =
+ new PluginInfo("transientSolrCoreCacheFactory",
+ ImmutableMap.of("class", TransientSolrCoreCacheFactoryDefault.class.getName(),
+ "name", TransientSolrCoreCacheFactory.class.getName()),
+ null, Collections.<PluginInfo>emptyList());
+
+
+ // Need this because the plugin framework doesn't require a PluginINfo in the init method, don't see a way to
+ // pass additional parameters and we need this when we create the transient core cache, it's _really_ important.
+ public void setCoreContainer(CoreContainer coreContainer) {
+ this.coreContainer = coreContainer;
+ }
+
+ public CoreContainer getCoreContainer() {
+ return coreContainer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactoryDefault.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactoryDefault.java b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactoryDefault.java
new file mode 100644
index 0000000..722ab9c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/TransientSolrCoreCacheFactoryDefault.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+public class TransientSolrCoreCacheFactoryDefault extends TransientSolrCoreCacheFactory {
+
+ TransientSolrCoreCache transientSolrCoreCache = null;
+
+ @Override
+ public TransientSolrCoreCache getTransientSolrCoreCache() {
+ if (transientSolrCoreCache == null) {
+ transientSolrCoreCache = new TransientSolrCoreCacheDefault(getCoreContainer());
+ }
+
+ return transientSolrCoreCache;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/test-files/solr/solr.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/solr.xml b/solr/core/src/test-files/solr/solr.xml
index f381475..526dffa 100644
--- a/solr/core/src/test-files/solr/solr.xml
+++ b/solr/core/src/test-files/solr/solr.xml
@@ -31,6 +31,11 @@
<int name="connTimeout">${connTimeout:15000}</int>
</shardHandlerFactory>
+ <transientCoreCacheFactory name="transientCoreCacheFactory" class="TransientSolrCoreCacheFactoryDefault">
+ <int name="transientCacheSize">4</int>
+ </transientCoreCacheFactory>
+
+
<solrcloud>
<str name="host">127.0.0.1</str>
<int name="hostPort">${hostPort:8983}</int>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
index d925774..d05cec9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
@@ -29,6 +29,8 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrXmlConfig;
+import org.apache.solr.core.TransientSolrCoreCache;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.update.UpdateShardHandler;
@@ -327,7 +329,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
private static class MockCoreContainer extends CoreContainer {
UpdateShardHandler updateShardHandler = new UpdateShardHandler(UpdateShardHandlerConfig.DEFAULT);
public MockCoreContainer() {
- super((Object)null);
+ super(SolrXmlConfig.fromString(null, "<solr/>"));
this.shardHandlerFactory = new HttpShardHandlerFactory();
this.coreAdminHandler = new CoreAdminHandler();
}
@@ -345,6 +347,11 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
updateShardHandler.close();
super.shutdown();
}
+
+ @Override
+ public TransientSolrCoreCache getTransientCacheHandler() {
+ return transientSolrCoreCache;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
index 65d459a..22020ba 100644
--- a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
+++ b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
@@ -60,12 +60,12 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
setMeUp(null);
}
- private Properties makeCorePropFile(String name, boolean isLazy, boolean loadOnStartup, String... extraProps) {
+ private Properties makeCorePropFile(String name, boolean isTransient, boolean loadOnStartup, String... extraProps) {
Properties props = new Properties();
props.put(CoreDescriptor.CORE_NAME, name);
props.put(CoreDescriptor.CORE_SCHEMA, "schema-tiny.xml");
props.put(CoreDescriptor.CORE_CONFIG, "solrconfig-minimal.xml");
- props.put(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(isLazy));
+ props.put(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(isTransient));
props.put(CoreDescriptor.CORE_LOADONSTARTUP, Boolean.toString(loadOnStartup));
props.put(CoreDescriptor.CORE_DATADIR, "${core.dataDir:stuffandnonsense}");
@@ -140,7 +140,7 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
try {
TestLazyCores.checkInCores(cc, "core1");
- TestLazyCores.checkNotInCores(cc, "lazy1", "core2", "collection1");
+ TestLazyCores.checkNotInCores(cc, "lazy1", "core2");
// force loading of core2 and lazy1 by getting them from the CoreContainer
try (SolrCore core1 = cc.getCore("core1");
@@ -463,4 +463,5 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
NodeConfig absConfig = SolrXmlConfig.fromString(loader, "<solr><str name=\"coreRootDirectory\">/absolute</str></solr>");
assertThat(absConfig.getCoreRootDirectory().toString(), not(containsString(solrHomeDirectory.getAbsolutePath())));
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
index 0c0845b..8690e27 100644
--- a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
+++ b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
@@ -38,6 +38,7 @@ import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.UpdateHandler;
@@ -83,13 +84,13 @@ public class TestLazyCores extends SolrTestCaseJ4 {
private CoreContainer init() throws Exception {
solrHomeDirectory = createTempDir().toFile();
+ copyXmlToHome(solrHomeDirectory.getAbsoluteFile(), "solr.xml");
for (int idx = 1; idx < 10; ++idx) {
copyMinConf(new File(solrHomeDirectory, "collection" + idx));
}
- SolrResourceLoader loader = new SolrResourceLoader(solrHomeDirectory.toPath());
- NodeConfig config = new NodeConfig.NodeConfigBuilder("testNode", loader).setTransientCacheSize(4).build();
- return createCoreContainer(config, testCores);
+ NodeConfig cfg = SolrDispatchFilter.loadNodeConfig(solrHomeDirectory.toPath(), null);
+ return createCoreContainer(cfg, testCores);
}
@Test
@@ -188,7 +189,7 @@ public class TestLazyCores extends SolrTestCaseJ4 {
, "//result[@numFound='0']"
);
- checkInCores(cc, "collection4");
+ checkInCores(cc, "collection1", "collection2", "collection4", "collection5");
core4.close();
collection1.close();
@@ -454,11 +455,14 @@ public class TestLazyCores extends SolrTestCaseJ4 {
// 1> produce errors as appropriate when the config or schema files are foo'd
// 2> "self heal". That is, if the problem is corrected can the core be reloaded and used?
// 3> that OK cores can be searched even when some cores failed to load.
+ // 4> that having no solr.xml entry for transient chache handler correctly uses the default.
@Test
public void testBadConfigsGenerateErrors() throws Exception {
final CoreContainer cc = initGoodAndBad(Arrays.asList("core1", "core2"),
Arrays.asList("badSchema1", "badSchema2"),
Arrays.asList("badConfig1", "badConfig2"));
+
+
try {
// first, did the two good cores load successfully?
checkInCores(cc, "core1", "core2");
@@ -491,8 +495,9 @@ public class TestLazyCores extends SolrTestCaseJ4 {
copyGoodConf("badSchema1", "schema-tiny.xml", "schema.xml");
copyGoodConf("badSchema2", "schema-tiny.xml", "schema.xml");
+
// This should force a reload of the cores.
- SolrCore bc1 = cc.getCore("badConfig1");
+ SolrCore bc1 = cc.getCore("badConfig1");;
SolrCore bc2 = cc.getCore("badConfig2");
SolrCore bs1 = cc.getCore("badSchema1");
SolrCore bs2 = cc.getCore("badSchema2");
@@ -635,16 +640,46 @@ public class TestLazyCores extends SolrTestCaseJ4 {
}
public static void checkNotInCores(CoreContainer cc, String... nameCheck) {
- Collection<String> names = cc.getCoreNames();
+ Collection<String> loadedNames = cc.getCoreNames();
for (String name : nameCheck) {
- assertFalse("core " + name + " was found in the list of cores", names.contains(name));
+ assertFalse("core " + name + " was found in the list of cores", loadedNames.contains(name));
+ }
+
+ // There was a problem at one point exacerbated by the poor naming conventions. So parallel to loaded cores, there
+ // should be the ability to get the core _names_ that are loaded as well as all the core names _possible_
+ //
+ // the names above should only contain loaded core names. Every name in names should be in allNames, but none of
+ // the names in nameCheck should be loaded and thus should not be in names.
+
+ Collection<String> allNames = cc.getAllCoreNames();
+ // Every core, loaded or not should be in the accumulated coredescriptors:
+ List<CoreDescriptor> descriptors = cc.getCoreDescriptors();
+
+ assertEquals("There should be as many coreDescriptors as coreNames", allNames.size(), descriptors.size());
+ for (CoreDescriptor desc : descriptors) {
+ assertTrue("Name should have a corresponding descriptor", allNames.contains(desc.getName()));
+ }
+
+ // First check that all loaded cores are in allNames.
+ for (String name : loadedNames) {
+ assertTrue("Loaded core " + name + " should have been found in the list of all possible core names",
+ allNames.contains(name));
+ }
+
+ for (String name : nameCheck) {
+ assertTrue("Not-currently-loaded core " + name + " should have been found in the list of all possible core names",
+ allNames.contains(name));
}
}
public static void checkInCores(CoreContainer cc, String... nameCheck) {
- Collection<String> names = cc.getCoreNames();
+ Collection<String> loadedNames = cc.getCoreNames();
+
+ assertEquals("There whould be exactly as many loaded cores as loaded names returned. ",
+ loadedNames.size(), nameCheck.length);
+
for (String name : nameCheck) {
- assertTrue("core " + name + " was not found in the list of cores", names.contains(name));
+ assertTrue("core " + name + " was not found in the list of cores", loadedNames.contains(name));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52632cfc/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index be8e96d..faf6707 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -2002,8 +2002,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
FileUtils.copyFile(new File(top, "solrconfig.snippet.randomindexconfig.xml"), new File(subHome, "solrconfig.snippet.randomindexconfig.xml"));
}
- // Creates minimal full setup, including the old solr.xml file that used to be hard coded in ConfigSolrXmlOld
- // TODO: remove for 5.0
+ // Creates minimal full setup, including solr.xml
public static void copyMinFullSetup(File dstRoot) throws IOException {
if (! dstRoot.exists()) {
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
@@ -2013,6 +2012,15 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
copyMinConf(dstRoot);
}
+ // Just copies the file indicated to the tmp home directory naming it "solr.xml"
+ public static void copyXmlToHome(File dstRoot, String fromFile) throws IOException {
+ if (! dstRoot.exists()) {
+ assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
+ }
+ File xmlF = new File(SolrTestCaseJ4.TEST_HOME(), fromFile);
+ FileUtils.copyFile(xmlF, new File(dstRoot, "solr.xml"));
+
+ }
// Creates a consistent configuration, _including_ solr.xml at dstRoot. Creates collection1/conf and copies
// the stock files in there.
@@ -2020,7 +2028,6 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
if (!dstRoot.exists()) {
assertTrue("Failed to make subdirectory ", dstRoot.mkdirs());
}
-
FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(dstRoot, "solr.xml"));
File subHome = new File(dstRoot, collection + File.separator + "conf");