You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2017/06/14 18:09:12 UTC
[2/5] lucene-solr:jira/SOLR-10834: SOLR-10876: Regression in loading
runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory
SOLR-10876: Regression in loading runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/92b17838
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/92b17838
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/92b17838
Branch: refs/heads/jira/SOLR-10834
Commit: 92b17838a346ad55a6a4ab796b8ab8cbbe4ffea2
Parents: f470bbc
Author: Noble Paul <no...@apache.org>
Authored: Wed Jun 14 18:07:40 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Wed Jun 14 18:07:40 2017 +0930
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../handler/dataimport/VariableResolver.java | 4 +-
.../java/org/apache/solr/core/PluginBag.java | 32 +++++++---
.../TemplateUpdateProcessorFactory.java | 18 +++---
.../processor/UpdateRequestProcessorChain.java | 17 ++---
.../configsets/cloud-minimal/conf/schema.xml | 1 +
.../processor/TemplateUpdateProcessorTest.java | 67 ++++++++++++++++++--
.../src/update-request-processors.adoc | 2 +-
.../cloud/AbstractFullDistribZkTestBase.java | 2 +-
9 files changed, 107 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a07b68f..4c35991 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -159,6 +159,8 @@ Bug Fixes
* SOLR-10830: Solr now correctly enforces that the '_root_' field has the same fieldType as the
uniqueKey field. With out this enforcement, child document updating was unreliable. (hossman)
+* SOLR-10876: Regression in loading runtime UpdateRequestProcessors like TemplateUpdateProcessorFactory (noble)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
----------------------------------------------------------------------
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
index 8fced58..090e21b 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
@@ -143,7 +143,7 @@ public class VariableResolver {
* @return the string with the placeholders replaced with their values
*/
public String replaceTokens(String template) {
- return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun);
+ return TemplateUpdateProcessorFactory.replaceTokens(template, cache, fun, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
}
public void addNamespace(String name, Map<String,Object> newMap) {
if (newMap != null) {
@@ -164,7 +164,7 @@ public class VariableResolver {
}
public List<String> getVariables(String expr) {
- return TemplateUpdateProcessorFactory.getVariables(expr, cache);
+ return TemplateUpdateProcessorFactory.getVariables(expr, cache, TemplateUpdateProcessorFactory.DOLLAR_BRACES_PLACEHOLDER_PATTERN);
}
static class CurrentLevel {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/core/src/java/org/apache/solr/core/PluginBag.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index e03fc06..b0c68f2 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -35,6 +35,9 @@ import java.util.zip.ZipInputStream;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.solr.api.Api;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.api.ApiSupport;
import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils;
@@ -46,15 +49,12 @@ import org.apache.solr.util.SimplePostTool;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
-import org.apache.solr.api.Api;
-import org.apache.solr.api.ApiBag;
-import org.apache.solr.api.ApiSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonMap;
-import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.api.ApiBag.HANDLER_NAME;
+import static org.apache.solr.common.params.CommonParams.NAME;
/**
* This manages the lifecycle of a set of plugin of the same type .
@@ -125,10 +125,12 @@ public class PluginBag<T> implements AutoCloseable {
public PluginHolder<T> createPlugin(PluginInfo info) {
if ("true".equals(String.valueOf(info.attributes.get("runtimeLib")))) {
log.debug(" {} : '{}' created with runtimeLib=true ", meta.getCleanTag(), info.name);
- return new LazyPluginHolder<>(meta, info, core, core.getMemClassLoader());
+ return new LazyPluginHolder<T>(meta, info, core, "true".equals(System.getProperty("enable.runtime.lib")) ?
+ core.getMemClassLoader() :
+ core.getResourceLoader(), true);
} else if ("lazy".equals(info.attributes.get("startup")) && meta.options.contains(SolrConfig.PluginOpts.LAZY)) {
log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
- return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader());
+ return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
} else {
T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
initInstance(inst, info);
@@ -371,11 +373,13 @@ public class PluginBag<T> implements AutoCloseable {
protected SolrException solrException;
private final SolrCore core;
protected ResourceLoader resourceLoader;
+ private final boolean isRuntimeLib;
- LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader) {
+ LazyPluginHolder(SolrConfig.SolrPluginInfo pluginMeta, PluginInfo pluginInfo, SolrCore core, ResourceLoader loader, boolean isRuntimeLib) {
super(pluginInfo);
this.pluginMeta = pluginMeta;
+ this.isRuntimeLib = isRuntimeLib;
this.core = core;
this.resourceLoader = loader;
if (loader instanceof MemClassLoader) {
@@ -413,7 +417,19 @@ public class PluginBag<T> implements AutoCloseable {
loader.loadJars();
}
Class<T> clazz = (Class<T>) pluginMeta.clazz;
- T localInst = core.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader);
+ T localInst = null;
+ try {
+ localInst = core.createInstance(pluginInfo.className, clazz, pluginMeta.getCleanTag(), null, resourceLoader);
+ } catch (SolrException e) {
+ if (isRuntimeLib && !(resourceLoader instanceof MemClassLoader)) {
+ throw new SolrException(SolrException.ErrorCode.getErrorCode(e.code()),
+ e.getMessage() + ". runtime library loading is not enabled, start Solr with -Denable.runtime.lib=true",
+ e.getCause());
+ }
+ throw e;
+
+
+ }
initInstance(localInst, pluginInfo);
if (localInst instanceof SolrCoreAware) {
SolrResourceLoader.assertAwareCompatibility(SolrCoreAware.class, localInst);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
index c16a0c7..19c331f 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.java
@@ -57,18 +57,18 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
doc.addField(fName, replaceTokens(template, templateCache, s -> {
Object v = doc.getFieldValue(s);
return v == null ? "" : v;
- }));
+ }, BRACES_PLACEHOLDER_PATTERN));
}
}
}
- public static Resolved getResolved(String template, Cache<String, Resolved> cache) {
+ public static Resolved getResolved(String template, Cache<String, Resolved> cache, Pattern pattern) {
Resolved r = cache == null ? null : cache.get(template);
if (r == null) {
r = new Resolved();
- Matcher m = PLACEHOLDER_PATTERN.matcher(template);
+ Matcher m = pattern.matcher(template);
while (m.find()) {
String variable = m.group(1);
r.startIndexes.add(m.start(0));
@@ -83,19 +83,19 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
/**
* Get a list of variables embedded in the template string.
*/
- public static List<String> getVariables(String template, Cache<String, Resolved> cache) {
- Resolved r = getResolved(template, cache);
+ public static List<String> getVariables(String template, Cache<String, Resolved> cache, Pattern pattern) {
+ Resolved r = getResolved(template, cache, pattern );
if (r == null) {
return Collections.emptyList();
}
return new ArrayList<>(r.variables);
}
- public static String replaceTokens(String template, Cache<String, Resolved> cache, Function<String, Object> fun) {
+ public static String replaceTokens(String template, Cache<String, Resolved> cache, Function<String, Object> fun, Pattern pattern) {
if (template == null) {
return null;
}
- Resolved r = getResolved(template, cache);
+ Resolved r = getResolved(template, cache, pattern);
if (r.startIndexes != null) {
StringBuilder sb = new StringBuilder(template);
for (int i = r.startIndexes.size() - 1; i >= 0; i--) {
@@ -115,6 +115,8 @@ public class TemplateUpdateProcessorFactory extends SimpleUpdateProcessorFactory
public List<String> variables = new ArrayList<>(2);
}
- public static final Pattern PLACEHOLDER_PATTERN = Pattern
+ public static final Pattern DOLLAR_BRACES_PLACEHOLDER_PATTERN = Pattern
.compile("[$][{](.*?)[}]");
+ public static final Pattern BRACES_PLACEHOLDER_PATTERN = Pattern
+ .compile("[{](.*?)[}]");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
index 05d1a5a..3db42aa 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
@@ -29,7 +29,6 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
@@ -272,16 +271,12 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
if (s.isEmpty()) continue;
UpdateRequestProcessorFactory p = core.getUpdateProcessors().get(s);
if (p == null) {
- try {
- PluginInfo pluginInfo = new PluginInfo("updateProcessor",
- Utils.makeMap("name", s,
- "class", s + "UpdateProcessorFactory",
- "runtimeLib", "true"));
-
- PluginBag.PluginHolder<UpdateRequestProcessorFactory> pluginHolder = core.getUpdateProcessors().createPlugin(pluginInfo);
- core.getUpdateProcessors().put(s, p = pluginHolder.get());
- } catch (SolrException e) {
- }
+ PluginInfo pluginInfo = new PluginInfo("updateProcessor",
+ Utils.makeMap("name", s,
+ "class", s + "UpdateProcessorFactory",
+ "runtimeLib", "true"));
+
+ core.getUpdateProcessors().put(s, p = core.getUpdateProcessors().createPlugin(pluginInfo).get());
if (p == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such processor " + s);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml
index 7b8b690..08a1716 100644
--- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml
+++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml
@@ -24,5 +24,6 @@
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="id" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="true" />
<uniqueKey>id</uniqueKey>
</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java
index 7ee8a34..e145219 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TemplateUpdateProcessorTest.java
@@ -17,22 +17,56 @@
package org.apache.solr.update.processor;
-import org.apache.solr.SolrTestCaseJ4;
+import java.lang.invoke.MethodHandles;
+
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TemplateUpdateProcessorTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(5)
+ .addConfig("conf1", configset("cloud-minimal"))
+ .configure();
+ }
+
+ @After
+ public void after() throws Exception {
+ cluster.deleteAllCollections();
+ cluster.shutdown();
+ }
+
+ @org.junit.Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
-public class TemplateUpdateProcessorTest extends SolrTestCaseJ4 {
public void testSimple() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams()
+ .add("processor", "Template")
+ .add("Template.field", "id:{firstName}_{lastName}")
+ .add("Template.field", "another:{lastName}_{firstName}")
+ .add("Template.field", "missing:{lastName}_{unKnown}");
AddUpdateCommand cmd = new AddUpdateCommand(new LocalSolrQueryRequest(null,
- new ModifiableSolrParams()
- .add("processor", "Template")
- .add("Template.field", "id:${firstName}_${lastName}")
- .add("Template.field", "another:${lastName}_${firstName}")
- .add("Template.field", "missing:${lastName}_${unKnown}")
+ params
));
cmd.solrDoc = new SolrInputDocument();
@@ -44,5 +78,24 @@ public class TemplateUpdateProcessorTest extends SolrTestCaseJ4 {
assertEquals("Cruise_Tom", cmd.solrDoc.getFieldValue("another"));
assertEquals("Cruise_", cmd.solrDoc.getFieldValue("missing"));
+ SolrInputDocument solrDoc = new SolrInputDocument();
+ solrDoc.addField("id", "1");
+
+ params = new ModifiableSolrParams()
+ .add("processor", "Template")
+ .add("commit", "true")
+ .add("Template.field", "x_s:key_{id}");
+ params.add("commit", "true");
+ UpdateRequest add = new UpdateRequest().add(solrDoc);
+ add.setParams(params);
+ NamedList<Object> result = cluster.getSolrClient().request(CollectionAdminRequest.createCollection("c", "conf1", 1, 1));
+ Utils.toJSONString(result.asMap(4));
+ AbstractFullDistribZkTestBase.waitForCollection(cluster.getSolrClient().getZkStateReader(), "c",1);
+ cluster.getSolrClient().request(add, "c");
+ QueryResponse rsp = cluster.getSolrClient().query("c",
+ new ModifiableSolrParams().add("q","id:1"));
+ assertEquals( "key_1", rsp.getResults().get(0).getFieldValue("x_s"));
+
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/solr-ref-guide/src/update-request-processors.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc
index a4f3e0c..d1f5c35 100644
--- a/solr/solr-ref-guide/src/update-request-processors.adoc
+++ b/solr/solr-ref-guide/src/update-request-processors.adoc
@@ -392,7 +392,7 @@ For example:
[source,bash]
----
-processor=Template&Template.field=fullName:Mr. ${firstName} ${lastName}
+processor=Template&Template.field=fullName:Mr. {firstName} {lastName}
----
The above example would add a new field to the document called `fullName`. The fields `firstName and` `lastName` are supplied from the document fields. If either of them is missing, that part is replaced with an empty string. If those fields are multi-valued, only the first value is used.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b17838/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index 5fa4af5..cf6f2e1 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -353,7 +353,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
}
- protected static void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception {
+ public static void waitForCollection(ZkStateReader reader, String collection, int slices) throws Exception {
// wait until shards have started registering...
int cnt = 30;
while (!reader.getClusterState().hasCollection(collection)) {