You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by st...@apache.org on 2012/05/21 11:07:03 UTC
svn commit: r1340941 - in /lucene/dev/branches/lucene_solr_3_6: ./ solr/
solr/contrib/ solr/contrib/clustering/
solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/
solr/contrib/clustering/src/test/org/apache/solr/handler/cluste...
Author: stanislaw
Date: Mon May 21 09:07:02 2012
New Revision: 1340941
URL: http://svn.apache.org/viewvc?rev=1340941&view=rev
Log:
SOLR-3470: Custom Carrot2 tokenizer and stemmer factories overwritten by defaults: fixing class loader issues, minor cleanups (Dawid Weiss), merge from trunk (r1340939)
Modified:
lucene/dev/branches/lucene_solr_3_6/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/contrib/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/CHANGES.txt
lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java
lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java
Modified: lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/CHANGES.txt?rev=1340941&r1=1340940&r2=1340941&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/CHANGES.txt Mon May 21 09:07:02 2012
@@ -10,7 +10,8 @@ $Id$
================== Release 3.6.1 ==================
-(No Changes)
+* SOLR-3470: Bug fix: custom Carrot2 tokenizer and stemmer factories are
+ respected now (Stanislaw Osinski, Dawid Weiss)
================== Release 3.6.0 ==================
Modified: lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java?rev=1340941&r1=1340940&r2=1340941&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java Mon May 21 09:07:02 2012
@@ -109,6 +109,9 @@ public class CarrotClusteringEngine exte
private Controller controller = ControllerFactory.createPooling();
private Class<? extends IClusteringAlgorithm> clusteringAlgorithmClass;
+ /** Solr core we're bound to. */
+ private SolrCore core;
+
private static class SolrResourceLocator implements IResourceLocator {
private final SolrResourceLoader resourceLoader;
private final String carrot2ResourcesDir;
@@ -145,7 +148,7 @@ public class CarrotClusteringEngine exte
public InputStream open() throws IOException {
return new ByteArrayInputStream(asBytes);
}
-
+
public int hashCode() {
// In case multiple resources are found they will be deduped, but we don't use it in Solr,
// so simply rely on instance equivalence.
@@ -217,8 +220,19 @@ public class CarrotClusteringEngine exte
extractCarrotAttributes(sreq.getParams(), attributes);
// Perform clustering and convert to named list
- return clustersToNamedList(controller.process(attributes,
- clusteringAlgorithmClass).getClusters(), sreq.getParams());
+ // Carrot2 uses current thread's context class loader to get
+ // certain classes (e.g. custom tokenizer/stemmer) at runtime.
+ // To make sure classes from contrib JARs are available,
+ // we swap the context class loader for the time of clustering.
+ Thread ct = Thread.currentThread();
+ ClassLoader prev = ct.getContextClassLoader();
+ try {
+ ct.setContextClassLoader(core.getResourceLoader().getClassLoader());
+ return clustersToNamedList(controller.process(attributes,
+ clusteringAlgorithmClass).getClusters(), sreq.getParams());
+ } finally {
+ ct.setContextClassLoader(prev);
+ }
} catch (Exception e) {
log.error("Carrot2 clustering failed", e);
throw new SolrException(ErrorCode.SERVER_ERROR, "Carrot2 clustering failed", e);
@@ -228,6 +242,8 @@ public class CarrotClusteringEngine exte
@Override
@SuppressWarnings({ "unchecked", "rawtypes" })
public String init(NamedList config, final SolrCore core) {
+ this.core = core;
+
String result = super.init(config, core);
final SolrParams initParams = SolrParams.toSolrParams(config);
@@ -262,8 +278,19 @@ public class CarrotClusteringEngine exte
// Using the class loader directly because this time we want to omit the prefix
new ClassLoaderLocator(core.getResourceLoader().getClassLoader())));
- this.controller.init(initAttributes);
-
+ // Carrot2 uses current thread's context class loader to get
+ // certain classes (e.g. custom tokenizer/stemmer) at initialization time.
+ // To make sure classes from contrib JARs are available,
+ // we swap the context class loader for the time of clustering.
+ Thread ct = Thread.currentThread();
+ ClassLoader prev = ct.getContextClassLoader();
+ try {
+ ct.setContextClassLoader(core.getResourceLoader().getClassLoader());
+ this.controller.init(initAttributes);
+ } finally {
+ ct.setContextClassLoader(prev);
+ }
+
SchemaField uniqueField = core.getSchema().getUniqueKeyField();
if (uniqueField == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
Modified: lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java?rev=1340941&r1=1340940&r2=1340941&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotParams.java Mon May 21 09:07:02 2012
@@ -22,29 +22,48 @@ import com.google.common.collect.Immutab
*/
-public interface CarrotParams {
+/**
+ * Carrot2 parameter mapping (recognized and mapped if passed via Solr configuration).
+ */
+public final class CarrotParams {
- String CARROT_PREFIX = "carrot.";
+ private static String CARROT_PREFIX = "carrot.";
- String ALGORITHM = CARROT_PREFIX + "algorithm";
+ public static String ALGORITHM = CARROT_PREFIX + "algorithm";
+
+ public static String TITLE_FIELD_NAME = CARROT_PREFIX + "title";
+ public static String URL_FIELD_NAME = CARROT_PREFIX + "url";
+ public static String SNIPPET_FIELD_NAME = CARROT_PREFIX + "snippet";
+ public static String LANGUAGE_FIELD_NAME = CARROT_PREFIX + "lang";
+ public static String CUSTOM_FIELD_NAME = CARROT_PREFIX + "custom";
- String TITLE_FIELD_NAME = CARROT_PREFIX + "title";
- String URL_FIELD_NAME = CARROT_PREFIX + "url";
- String SNIPPET_FIELD_NAME = CARROT_PREFIX + "snippet";
- String LANGUAGE_FIELD_NAME = CARROT_PREFIX + "lang";
- String CUSTOM_FIELD_NAME = CARROT_PREFIX + "custom";
+ public static String PRODUCE_SUMMARY = CARROT_PREFIX + "produceSummary";
+ public static String SUMMARY_FRAGSIZE = CARROT_PREFIX + "fragSize";
+ public static String SUMMARY_SNIPPETS = CARROT_PREFIX + "summarySnippets";
+
+ public static String NUM_DESCRIPTIONS = CARROT_PREFIX + "numDescriptions";
+ public static String OUTPUT_SUB_CLUSTERS = CARROT_PREFIX + "outputSubClusters";
+ public static String LEXICAL_RESOURCES_DIR = CARROT_PREFIX + "lexicalResourcesDir";
+ public static String LANGUAGE_CODE_MAP = CARROT_PREFIX + "lcmap";
+
+ static final Set<String> CARROT_PARAM_NAMES = ImmutableSet.of(
+ ALGORITHM,
+
+ TITLE_FIELD_NAME,
+ URL_FIELD_NAME,
+ SNIPPET_FIELD_NAME,
+ LANGUAGE_FIELD_NAME,
+ CUSTOM_FIELD_NAME,
+
+ PRODUCE_SUMMARY,
+ SUMMARY_FRAGSIZE,
+ SUMMARY_SNIPPETS,
+
+ NUM_DESCRIPTIONS,
+ OUTPUT_SUB_CLUSTERS,
+ LEXICAL_RESOURCES_DIR,
+ LANGUAGE_CODE_MAP);
- String PRODUCE_SUMMARY = CARROT_PREFIX + "produceSummary";
- String SUMMARY_FRAGSIZE = CARROT_PREFIX + "fragSize";
- String SUMMARY_SNIPPETS = CARROT_PREFIX + "summarySnippets";
-
- String NUM_DESCRIPTIONS = CARROT_PREFIX + "numDescriptions";
- String OUTPUT_SUB_CLUSTERS = CARROT_PREFIX + "outputSubClusters";
- String LEXICAL_RESOURCES_DIR = CARROT_PREFIX + "lexicalResourcesDir";
- String LANGUAGE_CODE_MAP = CARROT_PREFIX + "lcmap";
-
- public static final Set<String> CARROT_PARAM_NAMES = ImmutableSet.of(
- ALGORITHM, TITLE_FIELD_NAME, URL_FIELD_NAME, SNIPPET_FIELD_NAME, LANGUAGE_FIELD_NAME,
- PRODUCE_SUMMARY, SUMMARY_FRAGSIZE, SUMMARY_SNIPPETS, NUM_DESCRIPTIONS, OUTPUT_SUB_CLUSTERS,
- LEXICAL_RESOURCES_DIR);
+ /** No instances. */
+ private CarrotParams() {}
}
Modified: lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java?rev=1340941&r1=1340940&r2=1340941&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java Mon May 21 09:07:02 2012
@@ -205,7 +205,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void solrStopWordsUsedInCarrot2Clustering() throws Exception {
+ public void testSolrStopWordsUsedInCarrot2Clustering() throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("merge-resources", false);
params.set(AttributeUtils.getKey(
@@ -220,7 +220,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void solrStopWordsNotDefinedOnAFieldForClustering() throws Exception {
+ public void testSolrStopWordsNotDefinedOnAFieldForClustering() throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams();
// Force string fields to be used for clustering. Does not make sense
// in a real word, but does the job in the test.
@@ -239,7 +239,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void highlightingOfMultiValueField() throws Exception {
+ public void testHighlightingOfMultiValueField() throws Exception {
final String snippetWithoutSummary = getLabels(clusterWithHighlighting(
false, 30, 3, "multi", 1).get(0)).get(1);
assertTrue("Snippet contains first value", snippetWithoutSummary.contains("First"));
@@ -256,7 +256,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void concatenatingMultipleFields() throws Exception {
+ public void testConcatenatingMultipleFields() throws Exception {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CarrotParams.TITLE_FIELD_NAME, "title,heading");
params.add(CarrotParams.SNIPPET_FIELD_NAME, "snippet,body");
@@ -271,7 +271,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void highlightingMultipleFields() throws Exception {
+ public void testHighlightingMultipleFields() throws Exception {
final TermQuery query = new TermQuery(new Term("snippet", "content"));
final ModifiableSolrParams params = new ModifiableSolrParams();
@@ -297,7 +297,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void oneCarrot2SupportedLanguage() throws Exception {
+ public void testOneCarrot2SupportedLanguage() throws Exception {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CarrotParams.LANGUAGE_FIELD_NAME, "lang");
@@ -309,7 +309,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void oneCarrot2SupportedLanguageOfMany() throws Exception {
+ public void testOneCarrot2SupportedLanguageOfMany() throws Exception {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CarrotParams.LANGUAGE_FIELD_NAME, "lang");
@@ -321,7 +321,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void languageCodeMapping() throws Exception {
+ public void testLanguageCodeMapping() throws Exception {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CarrotParams.LANGUAGE_FIELD_NAME, "lang");
params.add(CarrotParams.LANGUAGE_CODE_MAP, "POLISH:pl");
@@ -334,7 +334,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void passingOfCustomFields() throws Exception {
+ public void testPassingOfCustomFields() throws Exception {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CarrotParams.CUSTOM_FIELD_NAME, "intfield_i:intfield");
params.add(CarrotParams.CUSTOM_FIELD_NAME, "floatfield_f:floatfield");
@@ -353,7 +353,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void customTokenizer() throws Exception {
+ public void testCustomTokenizer() throws Exception {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CarrotParams.TITLE_FIELD_NAME, "title");
params.add(CarrotParams.SNIPPET_FIELD_NAME, "snippet");
@@ -367,7 +367,7 @@ public class CarrotClusteringEngineTest
}
@Test
- public void customStemmer() throws Exception {
+ public void testCustomStemmer() throws Exception {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CarrotParams.TITLE_FIELD_NAME, "title");
params.add(CarrotParams.SNIPPET_FIELD_NAME, "snippet");