You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/08/10 14:54:46 UTC
lucene-solr:master: SOLR-9397: Config API does not support adding
caches
Repository: lucene-solr
Updated Branches:
refs/heads/master dd03d39dd -> 64c99293d
SOLR-9397: Config API does not support adding caches
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/64c99293
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/64c99293
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/64c99293
Branch: refs/heads/master
Commit: 64c99293d7d73c798c794cc647cf19636f62b2d6
Parents: dd03d39
Author: Noble Paul <no...@apache.org>
Authored: Wed Aug 10 20:23:22 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Wed Aug 10 20:24:28 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/core/SolrConfig.java | 29 ++++++--
.../org/apache/solr/search/CacheConfig.java | 24 ++++---
.../apache/solr/search/SolrIndexSearcher.java | 69 +++-----------------
.../apache/solr/core/TestSolrConfigHandler.java | 56 +++++++++++++++-
5 files changed, 106 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64c99293/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3451a4f..05252c3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -196,6 +196,8 @@ Bug Fixes
* SOLR-9179: NPE in IndexSchema using IBM JDK (noble, Colvin Cowie)
+* SOLR-9397: Config API does not support adding caches (noble)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64c99293/solr/core/src/java/org/apache/solr/core/SolrConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index eb3aa5f..653c612 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -28,7 +28,17 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -49,6 +59,7 @@ import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.search.CacheConfig;
import org.apache.solr.search.FastLRUCache;
import org.apache.solr.search.QParserPlugin;
+import org.apache.solr.search.SolrCache;
import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.search.stats.StatsCache;
import org.apache.solr.servlet.SolrRequestParsers;
@@ -91,7 +102,7 @@ public class SolrConfig extends Config implements MapSerializable {
public static final String DEFAULT_CONF_FILE = "solrconfig.xml";
private RequestParams requestParams;
- public static enum PluginOpts {
+ public enum PluginOpts {
MULTI_OK,
REQUIRE_NAME,
REQUIRE_NAME_IN_OVERLAY,
@@ -254,7 +265,6 @@ public class SolrConfig extends Config implements MapSerializable {
dataDir = get("dataDir", null);
if (dataDir != null && dataDir.length() == 0) dataDir = null;
- userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");
org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);
@@ -276,6 +286,16 @@ public class SolrConfig extends Config implements MapSerializable {
maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);
slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
+
+ Map<String, CacheConfig> userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");
+ List<PluginInfo> caches = getPluginInfos(SolrCache.class.getName());
+ if (!caches.isEmpty()) {
+ for (PluginInfo c : caches) {
+ userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null));
+ }
+ }
+ this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs);
+
updateHandlerInfo = loadUpdatehandlerInfo();
multipartUploadLimitKB = getInt(
@@ -317,6 +337,7 @@ public class SolrConfig extends Config implements MapSerializable {
.add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
.add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
+ .add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
// TODO: WTF is up with queryConverter???
// it apparently *only* works as a singleton? - SOLR-4304
// and even then -- only if there is a single SpellCheckComponent
@@ -457,7 +478,7 @@ public class SolrConfig extends Config implements MapSerializable {
public final CacheConfig queryResultCacheConfig;
public final CacheConfig documentCacheConfig;
public final CacheConfig fieldValueCacheConfig;
- public final CacheConfig[] userCacheConfigs;
+ public final Map<String, CacheConfig> userCacheConfigs;
// SolrIndexSearcher - more...
public final boolean useFilterForSortedQuery;
public final int queryResultWindowSize;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64c99293/solr/core/src/java/org/apache/solr/search/CacheConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
index 40e54dc..ee333f8 100644
--- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java
+++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
@@ -17,10 +17,10 @@
package org.apache.solr.search;
import javax.xml.xpath.XPathConstants;
-
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -75,14 +75,15 @@ public class CacheConfig implements MapSerializable{
this.regenerator = regenerator;
}
- public static CacheConfig[] getMultipleConfigs(SolrConfig solrConfig, String configPath) {
- NodeList nodes = (NodeList)solrConfig.evaluate(configPath, XPathConstants.NODESET);
- if (nodes==null || nodes.getLength()==0) return null;
- CacheConfig[] configs = new CacheConfig[nodes.getLength()];
- for (int i=0; i<nodes.getLength(); i++) {
- configs[i] = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()), configPath);
+ public static Map<String, CacheConfig> getMultipleConfigs(SolrConfig solrConfig, String configPath) {
+ NodeList nodes = (NodeList) solrConfig.evaluate(configPath, XPathConstants.NODESET);
+ if (nodes == null || nodes.getLength() == 0) return new LinkedHashMap<>();
+ Map<String, CacheConfig> result = new HashMap<>(nodes.getLength());
+ for (int i = 0; i < nodes.getLength(); i++) {
+ CacheConfig config = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()), configPath);
+ result.put(config.args.get(NAME), config);
}
- return configs;
+ return result;
}
@@ -101,9 +102,14 @@ public class CacheConfig implements MapSerializable{
public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map<String,String> attrs, String xpath) {
CacheConfig config = new CacheConfig();
config.nodeName = nodeName;
+ Map attrsCopy = new LinkedHashMap<>(attrs.size());
+ for (Map.Entry<String, String> e : attrs.entrySet()) {
+ attrsCopy.put(e.getKey(), String.valueOf(e.getValue()));
+ }
+ attrs = attrsCopy;
config.args = attrs;
- Map<String, String> map = solrConfig.getOverlay().getEditableSubProperties(xpath);
+ Map<String, String> map = xpath == null ? null : solrConfig.getOverlay().getEditableSubProperties(xpath);
if(map != null){
HashMap<String, String> mapCopy = new HashMap<>(config.args);
for (Map.Entry<String, String> e : map.entrySet()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64c99293/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index cc719f0..0f480c6 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -36,62 +36,16 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.collect.Iterables;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.document.LazyDocument;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.ExitableDirectoryReader;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiPostingsEnum;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.StoredFieldVisitor.Status;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.EarlyTerminatingSortingCollector;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MultiCollector;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SimpleCollector;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
-import org.apache.lucene.search.TimeLimitingCollector;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopDocsCollector;
-import org.apache.lucene.search.TopFieldCollector;
-import org.apache.lucene.search.TopFieldDocs;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.search.TotalHitCountCollector;
-import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -128,10 +82,6 @@ import org.apache.solr.update.SolrIndexConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.collect.Iterables;
-
/**
* SolrIndexSearcher adds schema awareness and caching functionality over {@link IndexSearcher}.
*
@@ -337,13 +287,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
if (documentCache != null) clist.add(documentCache);
- if (solrConfig.userCacheConfigs == null) {
+ if (solrConfig.userCacheConfigs.isEmpty()) {
cacheMap = NO_GENERIC_CACHES;
} else {
- cacheMap = new HashMap<>(solrConfig.userCacheConfigs.length);
- for (CacheConfig userCacheConfig : solrConfig.userCacheConfigs) {
- SolrCache cache = null;
- if (userCacheConfig != null) cache = userCacheConfig.newInstance();
+ cacheMap = new HashMap<>(solrConfig.userCacheConfigs.size());
+ for (Map.Entry<String,CacheConfig> e : solrConfig.userCacheConfigs.entrySet()) {
+ SolrCache cache = e.getValue().newInstance();
if (cache != null) {
cacheMap.put(cache.name(), cache);
clist.add(cache);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/64c99293/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
index 41e32dd..c182495 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
@@ -22,6 +22,7 @@ import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -35,8 +36,12 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.handler.DumpRequestHandler;
import org.apache.solr.handler.TestBlobHandler;
import org.apache.solr.handler.TestSolrConfigHandlerConcurrent;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.SolrCache;
import org.apache.solr.util.RestTestBase;
import org.apache.solr.util.RestTestHarness;
import org.eclipse.jetty.servlet.ServletHolder;
@@ -449,7 +454,56 @@ public class TestSolrConfigHandler extends RestTestBase {
assertEquals(2, initArgs.size());
assertTrue(((Map)initArgs.get(0)).containsKey("suggester"));
assertTrue(((Map)initArgs.get(1)).containsKey("suggester"));
- System.out.println(map);
+
+ payload = "{\n" +
+ "'add-requesthandler' : { 'name' : '/dump101', 'class': " +
+ "'" + CacheTest.class.getName() + "' " +
+ ", 'startup' : 'lazy'}\n" +
+ "}";
+ runConfigCommand(writeHarness, "/config?wt=json", payload);
+
+ testForResponseElement(writeHarness,
+ testServerBaseUrl,
+ "/config/overlay?wt=json",
+ cloudSolrClient,
+ Arrays.asList("overlay", "requestHandler", "/dump101", "startup"),
+ "lazy",
+ 10);
+
+ payload = "{\n" +
+ "'add-cache' : {name:'lfuCacheDecayFalse', class:'solr.search.LFUCache', size:10 ,initialSize:9 , timeDecay:false }," +
+ "'add-cache' : {name: 'perSegFilter', class: 'solr.search.LRUCache', size:10, initialSize:0 , autowarmCount:10}}";
+ runConfigCommand(writeHarness, "/config?wt=json", payload);
+
+ map = testForResponseElement(writeHarness,
+ testServerBaseUrl,
+ "/config/overlay?wt=json",
+ cloudSolrClient,
+ Arrays.asList("overlay", "cache", "lfuCacheDecayFalse", "class"),
+ "solr.search.LFUCache",
+ 10);
+ assertEquals("solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of("overlay", "cache", "perSegFilter", "class")));
+
+ map = getRespMap("/dump101?cacheNames=lfuCacheDecayFalse&cacheNames=perSegFilter&wt=json", writeHarness);
+ assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "perSegFilter")));
+ assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LFUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "lfuCacheDecayFalse")));
+
+ }
+
+ public static class CacheTest extends DumpRequestHandler {
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ super.handleRequestBody(req, rsp);
+ String[] caches = req.getParams().getParams("cacheNames");
+ if(caches != null && caches.length>0){
+ HashMap m = new HashMap();
+ rsp.add("caches", m);
+ for (String c : caches) {
+ SolrCache cache = req.getSearcher().getCache(c);
+ if(cache != null) m.put(c, cache.getClass().getName());
+ }
+ }
+ }
}
public static Map testForResponseElement(RestTestHarness harness,