You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/10/20 14:37:28 UTC
[1/2] lucene-solr:master: SOLR-9657: Use cache for templates
Repository: lucene-solr
Updated Branches:
refs/heads/master 97761966f -> a4952b11f
SOLR-9657: Use cache for templates
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/127bf9f7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/127bf9f7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/127bf9f7
Branch: refs/heads/master
Commit: 127bf9f772468cbc94478ad67d54652001b175e0
Parents: 14b6d93
Author: Noble Paul <no...@apache.org>
Authored: Thu Oct 20 20:06:21 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Thu Oct 20 20:06:21 2016 +0530
----------------------------------------------------------------------
.../handler/dataimport/VariableResolver.java | 4 +-
.../TemplateUpdateProcessorFactory.java | 13 +++--
.../java/org/apache/solr/common/util/Cache.java | 26 +++++++--
.../apache/solr/common/util/MapBackedCache.java | 57 ++++++++++++++++++++
.../solr/common/util/TestJavaBinCodec.java | 52 +-----------------
5 files changed, 92 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/127bf9f7/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
index f255657..51b5841 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
@@ -25,6 +25,8 @@ import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.solr.common.util.Cache;
+import org.apache.solr.common.util.MapBackedCache;
import org.apache.solr.update.processor.TemplateUpdateProcessorFactory;
import static org.apache.solr.update.processor.TemplateUpdateProcessorFactory.Resolved;
@@ -55,7 +57,7 @@ public class VariableResolver {
.compile("^(\\w*?)\\((.*?)\\)$");
private Map<String,Object> rootNamespace;
private Map<String,Evaluator> evaluators;
- private Map<String,Resolved> cache = new WeakHashMap<>();
+ private Cache<String,Resolved> cache = new MapBackedCache<>(new WeakHashMap<>());
private Function<String,Object> fun = this::resolve;
public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/127bf9f7/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
index dbe61b4..b791d3b 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
@@ -20,18 +20,21 @@ package org.apache.solr.update.processor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.Cache;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.util.ConcurrentLRUCache;
+
//Adds new fields to documents based on a template pattern specified via Template.field
// request parameters (multi-valued) or 'field' value specified in initArgs
public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory {
+ private Cache<String, Resolved> templateCache = new ConcurrentLRUCache<>(1000, 800, 900, 10, false, false, null);
@Override
protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp) {
String[] vals = getParams("field");
@@ -45,7 +48,7 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
String fName = val.substring(0, idx);
String template = val.substring(idx + 1);
- doc.addField(fName, replaceTokens(template, null, s -> {
+ doc.addField(fName, replaceTokens(template, templateCache, s -> {
Object v = doc.getFieldValue(s);
return v == null ? "" : v;
}));
@@ -55,7 +58,7 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
}
- public static Resolved getResolved(String template, Map<String, Resolved> cache) {
+ public static Resolved getResolved(String template, Cache<String, Resolved> cache) {
Resolved r = cache == null ? null : cache.get(template);
if (r == null) {
r = new Resolved();
@@ -74,7 +77,7 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
/**
* Get a list of variables embedded in the template string.
*/
- public static List<String> getVariables(String template, Map<String, Resolved> cache) {
+ public static List<String> getVariables(String template, Cache<String, Resolved> cache) {
Resolved r = getResolved(template, cache);
if (r == null) {
return Collections.emptyList();
@@ -82,7 +85,7 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
return new ArrayList<>(r.variables);
}
- public static String replaceTokens(String template, Map<String, Resolved> cache, Function<String, Object> fun) {
+ public static String replaceTokens(String template, Cache<String, Resolved> cache, Function<String, Object> fun) {
if (template == null) {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/127bf9f7/solr/solrj/src/java/org/apache/solr/common/util/Cache.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Cache.java b/solr/solrj/src/java/org/apache/solr/common/util/Cache.java
index 7be7f0c..2ec4d7e 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Cache.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Cache.java
@@ -16,13 +16,31 @@
*/
package org.apache.solr.common.util;
+import java.util.Objects;
+import java.util.function.Function;
+
public interface Cache<K, V> {
- public V put(K key, V val);
+ V put(K key, V val);
+
+ V get(K key);
+
+ V remove(K key);
- public V get(K key);
+ void clear();
- public V remove(K key);
+ default V computeIfAbsent(K key,
+ Function<? super K, ? extends V> mappingFunction) {
+ Objects.requireNonNull(mappingFunction);
+ V v;
+ if ((v = get(key)) == null) {
+ V newValue;
+ if ((newValue = mappingFunction.apply(key)) != null) {
+ put(key, newValue);
+ return newValue;
+ }
+ }
- public void clear();
+ return v;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/127bf9f7/solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java b/solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java
new file mode 100644
index 0000000..1aba20b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/util/MapBackedCache.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+
+import java.util.Map;
+import java.util.function.Function;
+
+// A cache backed by a map
+public class MapBackedCache<K, V> implements Cache<K, V> {
+
+ private final Map<K, V> map;
+
+ public MapBackedCache(Map<K, V> map) {
+ this.map = map;
+ }
+
+ @Override
+ public V put(K key, V val) {
+ return map.put(key, val);
+ }
+
+ @Override
+ public V get(K key) {
+ return map.get(key);
+ }
+
+ @Override
+ public V remove(K key) {
+ return map.remove(key);
+ }
+
+ @Override
+ public void clear() {
+ map.clear();
+ }
+
+ @Override
+ public V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
+ return map.computeIfAbsent(key, mappingFunction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/127bf9f7/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java
index 96ddc8b..e2229c5 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJavaBinCodec.java
@@ -340,30 +340,7 @@ public class TestJavaBinCodec extends SolrTestCaseJ4 {
assertTrue(l1.get(1).equals(l2.get(1)));
assertFalse(l1.get(1) == l2.get(1));
- JavaBinCodec.StringCache stringCache = new JavaBinCodec.StringCache(new Cache<JavaBinCodec.StringBytes, String>() {
- private HashMap<JavaBinCodec.StringBytes, String> cache = new HashMap<>();
-
- @Override
- public String put(JavaBinCodec.StringBytes key, String val) {
- return cache.put(key, val);
- }
-
- @Override
- public String get(JavaBinCodec.StringBytes key) {
- return cache.get(key);
- }
-
- @Override
- public String remove(JavaBinCodec.StringBytes key) {
- return cache.remove(key);
- }
-
- @Override
- public void clear() {
- cache.clear();
-
- }
- });
+ JavaBinCodec.StringCache stringCache = new JavaBinCodec.StringCache(new MapBackedCache<>(new HashMap<>()));
m1 = (Map) new JavaBinCodec(null, stringCache).unmarshal(new ByteArrayInputStream(b1));
@@ -409,32 +386,7 @@ public class TestJavaBinCodec extends SolrTestCaseJ4 {
Runtime.getRuntime().gc();
printMem("before cache init");
- Cache<JavaBinCodec.StringBytes, String> cache1 = new Cache<JavaBinCodec.StringBytes, String>() {
- private HashMap<JavaBinCodec.StringBytes, String> cache = new HashMap<>();
-
- @Override
- public String put(JavaBinCodec.StringBytes key, String val) {
- l.add(key);
- return cache.put(key, val);
-
- }
-
- @Override
- public String get(JavaBinCodec.StringBytes key) {
- return cache.get(key);
- }
-
- @Override
- public String remove(JavaBinCodec.StringBytes key) {
- return cache.remove(key);
- }
-
- @Override
- public void clear() {
- cache.clear();
-
- }
- };
+ Cache<JavaBinCodec.StringBytes, String> cache1 = new MapBackedCache<>(new HashMap<>()) ;
final JavaBinCodec.StringCache STRING_CACHE = new JavaBinCodec.StringCache(cache1);
// STRING_CACHE = new JavaBinCodec.StringCache(cache);
[2/2] lucene-solr:master: Merge remote-tracking branch 'origin/master'
Posted by no...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a4952b11
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a4952b11
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a4952b11
Branch: refs/heads/master
Commit: a4952b11fec9e366cd97010d3cc96ef2ce6e953a
Parents: 127bf9f 9776196
Author: Noble Paul <no...@apache.org>
Authored: Thu Oct 20 20:07:15 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Thu Oct 20 20:07:15 2016 +0530
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
.../apache/lucene/misc/SweetSpotSimilarity.java | 16 ++
solr/CHANGES.txt | 15 +-
solr/bin/solr | 18 +-
solr/bin/solr.cmd | 28 +-
.../src/java/org/apache/solr/util/SolrCLI.java | 273 ++++++++++++++++++-
.../org/apache/solr/util/UtilsToolTest.java | 185 +++++++++++++
7 files changed, 499 insertions(+), 38 deletions(-)
----------------------------------------------------------------------