You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2013/09/08 14:21:56 UTC
svn commit: r1520859 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/classification/ lucene/classification/src/
lucene/code...
Author: tommaso
Date: Sun Sep 8 12:21:55 2013
New Revision: 1520859
URL: http://svn.apache.org/r1520859
Log:
SOLR-5201 - patch backported to branch_4x
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ (props changed)
lucene/dev/branches/branch_4x/lucene/classification/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ivy.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/src/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/replicator/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java
lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java
lucene/dev/branches/branch_4x/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java?rev=1520859&r1=1520858&r2=1520859&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java Sun Sep 8 12:21:55 2013
@@ -17,49 +17,49 @@ package org.apache.solr.uima.processor;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
-import org.apache.lucene.analysis.uima.ae.AEProvider;
-import org.apache.lucene.analysis.uima.ae.AEProviderFactory;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.JCasPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Map;
-
/**
* Update document(s) to be indexed with UIMA extracted information
- *
+ *
*/
public class UIMAUpdateRequestProcessor extends UpdateRequestProcessor {
-
- private final Logger log = LoggerFactory.getLogger(UIMAUpdateRequestProcessor.class);
-
+
+ private final Logger log = LoggerFactory
+ .getLogger(UIMAUpdateRequestProcessor.class);
+
SolrUIMAConfiguration solrUIMAConfiguration;
-
- private AEProvider aeProvider;
-
- public UIMAUpdateRequestProcessor(UpdateRequestProcessor next, String coreName,
- SolrUIMAConfiguration config) {
+
+ private AnalysisEngine ae;
+
+ private JCasPool pool;
+
+ public UIMAUpdateRequestProcessor(UpdateRequestProcessor next,
+ String coreName, SolrUIMAConfiguration config, AnalysisEngine ae,
+ JCasPool pool) {
super(next);
- initialize(coreName, config);
- }
-
- private void initialize(String coreName, SolrUIMAConfiguration config) {
+ this.ae = ae;
+ this.pool = pool;
solrUIMAConfiguration = config;
- aeProvider = AEProviderFactory.getInstance().getAEProvider(coreName,
- solrUIMAConfiguration.getAePath(), solrUIMAConfiguration.getRuntimeParameters());
}
-
+
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
String text = null;
@@ -72,54 +72,66 @@ public class UIMAUpdateRequestProcessor
for (int i = 0; i < texts.length; i++) {
text = texts[i];
if (text != null && text.length()>0) {
- /* process the text value */
- JCas jcas = processText(text);
-
- UIMAToSolrMapper uimaToSolrMapper = new UIMAToSolrMapper(solrInputDocument, jcas);
- /* get field mapping from config */
- Map<String, Map<String, MapField>> typesAndFeaturesFieldsMap = solrUIMAConfiguration
- .getTypesFeaturesFieldsMapping();
- /* map type features on fields */
- for (String typeFQN : typesAndFeaturesFieldsMap.keySet()) {
- uimaToSolrMapper.map(typeFQN, typesAndFeaturesFieldsMap.get(typeFQN));
+ /* create a JCas which contain the text to analyze */
+ JCas jcas = pool.getJCas(0);
+ try {
+ /* process the text value */
+ processText(text, jcas);
+
+ UIMAToSolrMapper uimaToSolrMapper = new UIMAToSolrMapper(
+ solrInputDocument, jcas);
+ /* get field mapping from config */
+ Map<String,Map<String,MapField>> typesAndFeaturesFieldsMap = solrUIMAConfiguration
+ .getTypesFeaturesFieldsMapping();
+ /* map type features on fields */
+ for (Entry<String,Map<String,MapField>> entry : typesAndFeaturesFieldsMap
+ .entrySet()) {
+ uimaToSolrMapper.map(entry.getKey(), entry.getValue());
+ }
+ } finally {
+ pool.releaseJCas(jcas);
}
}
}
} catch (Exception e) {
String logField = solrUIMAConfiguration.getLogField();
- if(logField == null){
- SchemaField uniqueKeyField = cmd.getReq().getSchema().getUniqueKeyField();
- if(uniqueKeyField != null){
+ if (logField == null) {
+ SchemaField uniqueKeyField = cmd.getReq().getSchema()
+ .getUniqueKeyField();
+ if (uniqueKeyField != null) {
logField = uniqueKeyField.getName();
}
}
- String optionalFieldInfo = logField == null ? "." :
- new StringBuilder(". ").append(logField).append("=")
- .append((String)cmd.getSolrInputDocument().getField(logField).getValue())
- .append(", ").toString();
+ String optionalFieldInfo = logField == null ? "."
+ : new StringBuilder(". ")
+ .append(logField)
+ .append("=")
+ .append(
+ (String) cmd.getSolrInputDocument().getField(logField)
+ .getValue()).append(", ").toString();
int len;
String debugString;
if (text != null && text.length() > 0) {
len = Math.min(text.length(), 100);
- debugString = new StringBuilder(" text=\"").append(text.substring(0, len)).append("...\"").toString();
- }
- else {
+ debugString = new StringBuilder(" text=\"")
+ .append(text.substring(0, len)).append("...\"").toString();
+ } else {
debugString = " null text";
}
if (solrUIMAConfiguration.isIgnoreErrors()) {
- log.warn("skip the text processing due to {}",new StringBuilder()
- .append(e.getLocalizedMessage()).append(optionalFieldInfo)
- .append(debugString));
+ log.warn(
+ "skip the text processing due to {}",
+ new StringBuilder().append(e.getLocalizedMessage())
+ .append(optionalFieldInfo).append(debugString));
} else {
- throw new SolrException(ErrorCode.SERVER_ERROR,
- new StringBuilder("processing error ")
- .append(e.getLocalizedMessage()).append(optionalFieldInfo)
- .append(debugString).toString(), e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, new StringBuilder(
+ "processing error ").append(e.getLocalizedMessage())
+ .append(optionalFieldInfo).append(debugString).toString(), e);
}
}
super.processAdd(cmd);
}
-
+
/*
* get the texts to analyze from the corresponding fields
*/
@@ -129,31 +141,32 @@ public class UIMAUpdateRequestProcessor
String[] textVals;
if (merge) {
StringBuilder unifiedText = new StringBuilder("");
- for (int i = 0; i < fieldsToAnalyze.length; i++) {
- unifiedText.append(String.valueOf(solrInputDocument.getFieldValue(fieldsToAnalyze[i])));
+ for (String aFieldsToAnalyze : fieldsToAnalyze) {
+ unifiedText.append(String.valueOf(solrInputDocument
+ .getFieldValue(aFieldsToAnalyze)));
}
textVals = new String[1];
textVals[0] = unifiedText.toString();
} else {
textVals = new String[fieldsToAnalyze.length];
for (int i = 0; i < fieldsToAnalyze.length; i++) {
- textVals[i] = String.valueOf(solrInputDocument.getFieldValue(fieldsToAnalyze[i]));
+ textVals[i] = String.valueOf(solrInputDocument
+ .getFieldValue(fieldsToAnalyze[i]));
}
}
return textVals;
}
-
- /* process a field value executing UIMA the CAS containing it as document text */
- private JCas processText(String textFieldValue) throws ResourceInitializationException,
- AnalysisEngineProcessException {
+
+ /*
+ * process a field value executing UIMA on the JCas containing it as document
+ * text
+ */
+ private void processText(String textFieldValue, JCas jcas)
+ throws ResourceInitializationException, AnalysisEngineProcessException {
if (log.isDebugEnabled()) {
log.debug("Analyzing text");
}
- /* get the UIMA analysis engine */
- AnalysisEngine ae = aeProvider.getAE();
- /* create a JCas which contain the text to analyze */
- JCas jcas = ae.newJCas();
jcas.setDocumentText(textFieldValue);
/* perform analysis on text field */
@@ -161,7 +174,6 @@ public class UIMAUpdateRequestProcessor
if (log.isDebugEnabled()) {
log.debug("Text processing completed");
}
- return jcas;
}
-
+
}
Modified: lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java?rev=1520859&r1=1520858&r2=1520859&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorFactory.java Sun Sep 8 12:21:55 2013
@@ -17,20 +17,29 @@ package org.apache.solr.uima.processor;
* limitations under the License.
*/
+import org.apache.lucene.analysis.uima.ae.AEProvider;
+import org.apache.lucene.analysis.uima.ae.AEProviderFactory;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.uima.analysis_engine.AnalysisEngine;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.JCasPool;
/**
* Factory for {@link UIMAUpdateRequestProcessor}
*
- *
+ *
*/
-public class UIMAUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
+public class UIMAUpdateRequestProcessorFactory extends
+ UpdateRequestProcessorFactory {
private NamedList<Object> args;
+ private AnalysisEngine ae;
+ private JCasPool pool;
@SuppressWarnings("unchecked")
@Override
@@ -39,10 +48,26 @@ public class UIMAUpdateRequestProcessorF
}
@Override
- public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp,
- UpdateRequestProcessor next) {
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ SolrUIMAConfiguration configuration = new SolrUIMAConfigurationReader(args)
+ .readSolrUIMAConfiguration();
+ synchronized (this) {
+ if (ae == null && pool == null) {
+ AEProvider aeProvider = AEProviderFactory.getInstance().getAEProvider(
+ req.getCore().getName(), configuration.getAePath(),
+ configuration.getRuntimeParameters());
+ try {
+ ae = aeProvider.getAE();
+ pool = new JCasPool(10, ae);
+ } catch (ResourceInitializationException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+ }
+
return new UIMAUpdateRequestProcessor(next, req.getCore().getName(),
- new SolrUIMAConfigurationReader(args).readSolrUIMAConfiguration());
+ configuration, ae, pool);
}
-
+
}
Modified: lucene/dev/branches/branch_4x/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1520859&r1=1520858&r2=1520859&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Sun Sep 8 12:21:55 2013
@@ -93,7 +93,6 @@ public class UIMAUpdateRequestProcessorT
@Test
public void testProcessing() throws Exception {
-
addDoc("uima", adoc(
"id",
"2312312321312",
@@ -185,6 +184,13 @@ public class UIMAUpdateRequestProcessorT
}
}
+ @Test
+ public void testMultiplierProcessing() throws Exception {
+ for (int i = 0; i < RANDOM_MULTIPLIER; i++) {
+ testProcessing();
+ }
+ }
+
private void addDoc(String chain, String doc) throws Exception {
Map<String, String[]> params = new HashMap<String, String[]>();
params.put(UpdateParams.UPDATE_CHAIN, new String[] { chain });