You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/07/25 19:41:23 UTC
svn commit: r1365673 [3/3] - in /lucene/dev/branches/branch_4x: ./
dev-tools/ dev-tools/eclipse/ lucene/ lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/
lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/ luce...
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Wed Jul 25 17:41:09 2012
@@ -88,32 +88,6 @@ final class TestRuleSetupAndRestoreClass
restoreProperties.put("solr.solr.home", System.getProperty("solr.solr.home"));
restoreProperties.put("solr.data.dir", System.getProperty("solr.data.dir"));
- // enable the Lucene 3.x PreflexRW codec explicitly, to work around bugs in IBM J9 / Harmony ServiceLoader:
- try {
- final java.lang.reflect.Field spiLoaderField = Codec.class.getDeclaredField("loader");
- spiLoaderField.setAccessible(true);
- final Object spiLoader = spiLoaderField.get(null);
- final java.lang.reflect.Field modifiableServicesField = NamedSPILoader.class.getDeclaredField("modifiableServices");
- modifiableServicesField.setAccessible(true);
- @SuppressWarnings({"unchecked","rawtypes"}) final Map<String,Codec> serviceMap =
- (Map) modifiableServicesField.get(spiLoader);
- if (!(Codec.forName("Lucene3x") instanceof PreFlexRWCodec)) {
- if (Constants.JAVA_VENDOR.startsWith("IBM")) {
- // definitely a buggy version
- System.err.println("ERROR: Your VM's java.util.ServiceLoader implementation is buggy"+
- " and does not respect classpath order, please report this to the vendor.");
- } else {
- // could just be a classpath issue
- System.err.println("ERROR: fix your classpath to have tests-framework.jar before lucene-core.jar!"+
- " If you have already done this, then your VM's java.util.ServiceLoader implementation is buggy"+
- " and does not respect classpath order, please report this to the vendor.");
- }
- serviceMap.put("Lucene3x", new PreFlexRWCodec());
- }
- } catch (Exception e) {
- throw new RuntimeException("Cannot access internals of Codec and NamedSPILoader classes", e);
- }
-
// if verbose: print some debugging stuff about which codecs are loaded.
if (VERBOSE) {
Set<String> codecs = Codec.availableCodecs();
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Jul 25 17:41:09 2012
@@ -77,6 +77,13 @@ New Features
already exist. To assert that the document must exist, use the optimistic
concurrency feature by specifying a _version_ of 1. (yonik)
+* LUCENE-2510, LUCENE-4044: Migrated Solr's Tokenizer-, TokenFilter-, and
+ CharFilterFactories to the lucene-analysis module. To add new analysis
+ modules to Solr (like ICU, SmartChinese, Morfologik,...), just drop in
+ the JAR files from Lucene's binary distribution into your Solr instance's
+ lib folder. The factories are automatically made available with SPI.
+ (Chris Male, Robert Muir, Uwe Schindler)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/SolrStopwordsCarrot2LexicalDataFactory.java Wed Jul 25 17:41:09 2012
@@ -23,8 +23,8 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import org.apache.solr.analysis.CommonGramsFilterFactory;
-import org.apache.solr.analysis.StopFilterFactory;
+import org.apache.lucene.analysis.commongrams.CommonGramsFilterFactory;
+import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.schema.IndexSchema;
import org.carrot2.core.LanguageCode;
Modified: lucene/dev/branches/branch_4x/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml (original)
+++ lucene/dev/branches/branch_4x/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml Wed Jul 25 17:41:09 2012
@@ -299,14 +299,14 @@
<fieldType name="uima_sentences" class="solr.TextField" positionIncrementGap="100">
<analyzer>
- <tokenizer class="org.apache.solr.uima.analysis.UIMAAnnotationsTokenizerFactory"
+ <tokenizer class="solr.UIMAAnnotationsTokenizerFactory"
descriptorPath="/uima/AggregateSentenceAE.xml" tokenType="org.apache.uima.SentenceAnnotation"/>
</analyzer>
</fieldType>
<fieldType name="uima_nouns" class="solr.TextField" positionIncrementGap="100">
<analyzer>
- <tokenizer class="org.apache.solr.uima.analysis.UIMATypeAwareAnnotationsTokenizerFactory"
+ <tokenizer class="solr.UIMATypeAwareAnnotationsTokenizerFactory"
descriptorPath="/uima/AggregateSentenceAE.xml" tokenType="org.apache.uima.TokenAnnotation"
featurePath="posTag"/>
<filter class="solr.TypeTokenFilterFactory" types="uima/stoptypes.txt" />
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java Wed Jul 25 17:41:09 2012
@@ -20,6 +20,7 @@ package org.apache.solr.analysis;
import java.io.Reader;
+import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
/**
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Wed Jul 25 17:41:09 2012
@@ -29,11 +29,15 @@ import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.analysis.util.AnalysisSPILoader;
+import org.apache.lucene.util.WeakIdentityMap;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
@@ -366,8 +370,15 @@ public class SolrResourceLoader implemen
/*
* A static map of short class name to fully qualified class name
*/
- private static Map<String, String> classNameCache = new ConcurrentHashMap<String, String>();
+ private static final Map<String, String> classNameCache = new ConcurrentHashMap<String, String>();
+ // A static map of AnalysisSPILoaders, keyed by ClassLoader used (because it can change during Solr lifetime) and expected base class:
+ private static final WeakIdentityMap<ClassLoader, Map<Class<?>,AnalysisSPILoader<?>>> expectedTypesSPILoaders = WeakIdentityMap.newConcurrentHashMap();
+
+ // Using this pattern, legacy analysis components from previous Solr versions are identified and delegated to SPI loader:
+ private static final Pattern legacyAnalysisPattern =
+ Pattern.compile("((\\Q"+base+".analysis.\\E)|(\\Q"+project+".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
+
/**
* This method loads a class either with it's FQN or a short-name (solr.class-simplename or class-simplename).
* It tries to load the class with the name that is given first and if it fails, it tries all the known
@@ -394,6 +405,31 @@ public class SolrResourceLoader implemen
}
}
Class<? extends T> clazz = null;
+
+ // first try legacy analysis patterns, now replaced by Lucene's Analysis package:
+ final Matcher m = legacyAnalysisPattern.matcher(cname);
+ if (m.matches()) {
+ log.trace("Trying to load class from analysis SPI");
+ // retrieve the map of classLoader -> expectedType -> SPI from cache / regenerate cache
+ Map<Class<?>,AnalysisSPILoader<?>> spiLoaders = expectedTypesSPILoaders.get(classLoader);
+ if (spiLoaders == null) {
+ spiLoaders = new IdentityHashMap<Class<?>,AnalysisSPILoader<?>>(3);
+ spiLoaders.put(CharFilterFactory.class, CharFilterFactory.getSPILoader(classLoader));
+ spiLoaders.put(TokenizerFactory.class, TokenizerFactory.getSPILoader(classLoader));
+ spiLoaders.put(TokenFilterFactory.class, TokenFilterFactory.getSPILoader(classLoader));
+ expectedTypesSPILoaders.put(classLoader, spiLoaders);
+ }
+ final AnalysisSPILoader<?> loader = spiLoaders.get(expectedType);
+ if (loader != null) {
+ // it's a correct expected type for analysis! Let's go on!
+ try {
+ return clazz = loader.lookupClass(m.group(4)).asSubclass(expectedType);
+ } catch (IllegalArgumentException ex) {
+ // ok, we fall back to legacy loading
+ }
+ }
+ }
+
// first try cname == full name
try {
return Class.forName(cname, true, classLoader).asSubclass(expectedType);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java Wed Jul 25 17:41:09 2012
@@ -19,9 +19,9 @@ package org.apache.solr.schema;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
import org.apache.lucene.analysis.util.*;
import org.apache.lucene.util.Version;
-import org.apache.solr.analysis.KeywordTokenizerFactory;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.DOMUtil;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java Wed Jul 25 17:41:09 2012
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.queries.function.BoostedQuery;
import org.apache.lucene.queries.function.FunctionQuery;
@@ -36,7 +37,6 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
-import org.apache.solr.analysis.StopFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.search.SolrQueryParser.MagicFieldName;
import org.apache.solr.common.params.DisMaxParams;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java Wed Jul 25 17:41:09 2012
@@ -1,3 +1,5 @@
+package org.apache.solr.analysis;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,8 +17,6 @@
* limitations under the License.
*/
-package org.apache.solr.analysis;
-
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilterFactory;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
import org.junit.BeforeClass;
@@ -33,6 +34,7 @@ import org.junit.Test;
/**
* New WordDelimiterFilter tests... most of the tests are in ConvertedLegacyTest
*/
+// TODO: add a low-level test for this factory
public class TestWordDelimiterFilterFactory extends SolrTestCaseJ4 {
@BeforeClass
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/ResourceLoaderTest.java Wed Jul 25 17:41:09 2012
@@ -20,8 +20,8 @@ package org.apache.solr.core;
import junit.framework.Assert;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.analysis.KeywordTokenizerFactory;
-import org.apache.solr.analysis.NGramFilterFactory;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
+import org.apache.lucene.analysis.ngram.NGramFilterFactory;
import org.apache.solr.common.SolrException;
import org.apache.solr.handler.admin.LukeRequestHandler;
import org.apache.solr.handler.component.FacetComponent;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java?rev=1365673&r1=1365672&r2=1365673&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java Wed Jul 25 17:41:09 2012
@@ -18,6 +18,12 @@ package org.apache.solr.schema;
*/
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
+import org.apache.lucene.analysis.core.LowerCaseFilterFactory;
+import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
+import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
+import org.apache.lucene.analysis.miscellaneous.TrimFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.analysis.*;