You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/05/03 15:54:45 UTC
svn commit: r940459 - in /lucene/dev/trunk: lucene/ lucene/contrib/
lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/
lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/
lucene/contrib/analyzers/common/src/test...
Author: rmuir
Date: Mon May 3 13:54:44 2010
New Revision: 940459
URL: http://svn.apache.org/viewvc?rev=940459&view=rev
Log:
LUCENE-2413: consolidate PorterStemmer to contrib/analyzers
Added:
lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemFilter.java
- copied, changed from r937139, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java
lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java
- copied, changed from r937139, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemmer.java
lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
- copied, changed from r937139, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java
lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/porterTestData.zip
- copied unchanged from r937139, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/porterTestData.zip
Removed:
lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/AnalyzerUtil.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemmer.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/porterTestData.zip
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/contrib/CHANGES.txt
lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java
lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
lucene/dev/trunk/lucene/contrib/analyzers/smartcn/build.xml
lucene/dev/trunk/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java
lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java
lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java
lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
lucene/dev/trunk/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon May 3 13:54:44 2010
@@ -4,6 +4,10 @@ Lucene Change Log
Changes in backwards compatibility policy
+* LUCENE-2413: Consolidated all Lucene analyzers into contrib/analyzers.
+ - o.a.l.analysis.PorterStemFilter -> o.a.l.analysis.en.PorterStemFilter
+ ... (in progress)
+
* LUCENE-1458, LUCENE-2111, LUCENE-2354: Changes from flexible indexing:
- On upgrading to 3.1, if you do not fully reindex your documents,
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Mon May 3 13:54:44 2010
@@ -23,6 +23,11 @@ Changes in backwards compatibility polic
* LUCENE-2323: Moved contrib/wikipedia functionality into contrib/analyzers.
Additionally the package was changed from org.apache.lucene.wikipedia.analysis
to org.apache.lucene.analysis.wikipedia. (Robert Muir)
+
+ * LUCENE-2413: Consolidated all analyzers into contrib/analyzers.
+ - contrib/analyzers/smartcn now depends on contrib/analyzers/common
+ - The "AnalyzerUtil" in wordnet was removed.
+ ... (in progress)
Changes in runtime behavior
Modified: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java Mon May 3 13:54:44 2010
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.KeywordMarkerFilter;
import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemFilter.java (from r937139, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemFilter.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java&r1=937139&r2=940459&rev=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemFilter.java Mon May 3 13:54:44 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.en;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -19,6 +19,8 @@ package org.apache.lucene.analysis;
import java.io.IOException;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java (from r937139, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemmer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemmer.java&r1=937139&r2=940459&rev=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemmer.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java Mon May 3 13:54:44 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.en;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (from r937139, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java&r1=937139&r2=940459&rev=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java Mon May 3 13:54:44 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.en;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -24,6 +24,14 @@ import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.zip.ZipFile;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.KeywordMarkerFilter;
+import org.apache.lucene.analysis.KeywordTokenizer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+
/**
* Test the PorterStemFilter with Martin Porter's test data.
*/
Modified: lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java Mon May 3 13:54:44 2010
@@ -7,7 +7,7 @@ import java.util.Map;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.KeywordTokenizer;
-import org.apache.lucene.analysis.PorterStemFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
Modified: lucene/dev/trunk/lucene/contrib/analyzers/smartcn/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/smartcn/build.xml?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/analyzers/smartcn/build.xml (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/smartcn/build.xml Mon May 3 13:54:44 2010
@@ -28,11 +28,27 @@
<property name="maven.dist.dir" location="../../../dist/maven" />
<import file="../../contrib-build.xml"/>
-
+
+ <contrib-uptodate name="analyzers/common" jarfile="${common.dir}/build/contrib/analyzers/common/lucene-analyzers-${version}.jar"
+ property="analyzers-common.uptodate" classpath.property="analyzers-common.jar"/>
+
+ <path id="classpath">
+ <pathelement path="${analyzers-common.jar}"/>
+ <path refid="base.classpath"/>
+ </path>
+
<path id="test.classpath">
+ <pathelement path="${analyzers-common.jar}"/>
<path refid="classpath"/>
<pathelement location="../../../build/classes/test/"/>
<path refid="junit-path"/>
<pathelement location="${build.dir}/classes/java"/>
- </path>
+ </path>
+
+ <target name="compile-core" depends="build-analyzers-common, common.compile-core" />
+
+ <target name="build-analyzers-common" unless="analyzers-common.uptodate">
+ <echo>Misc building dependency ${analyzers-common.jar}</echo>
+ <ant antfile="../common/build.xml" target="default" inheritall="false" dir="../common" />
+ </target>
</project>
Modified: lucene/dev/trunk/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java Mon May 3 13:54:44 2010
@@ -25,7 +25,7 @@ import java.util.Collections;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.PorterStemFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/PorterStemFilterFactory.java Mon May 3 13:54:44 2010
@@ -18,7 +18,7 @@
package org.apache.solr.analysis;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.PorterStemFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
/**
* @version $Id$
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java Mon May 3 13:54:44 2010
@@ -23,7 +23,7 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.analysis.PorterStemFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WhitespaceTokenizer;
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java Mon May 3 13:54:44 2010
@@ -23,7 +23,7 @@ import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.analysis.PorterStemFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WhitespaceTokenizer;
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java Mon May 3 13:54:44 2010
@@ -190,7 +190,7 @@ public class DocumentAnalysisRequestHand
assertNotNull("Expecting the 'StopFilter' to be applied on the query for the 'text' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
assertToken(tokenList.get(0), new TokenInfo("jumping", null, "<ALPHANUM>", 0, 7, 1, null, false));
- tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.PorterStemFilter");
+ tokenList = (List<NamedList>) queryResult.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expecting the 'PorterStemFilter' to be applied on the query for the 'text' field", tokenList);
assertEquals("Query has only one token", 1, tokenList.size());
assertToken(tokenList.get(0), new TokenInfo("jump", null, "<ALPHANUM>", 0, 7, 1, null, false));
@@ -231,7 +231,7 @@ public class DocumentAnalysisRequestHand
assertToken(tokenList.get(1), new TokenInfo("jumped", null, "<ALPHANUM>", 8, 14, 2, null, false));
assertToken(tokenList.get(2), new TokenInfo("over", null, "<ALPHANUM>", 15, 19, 3, null, false));
assertToken(tokenList.get(3), new TokenInfo("dogs", null, "<ALPHANUM>", 24, 28, 4, null, false));
- tokenList = valueResult.get("org.apache.lucene.analysis.PorterStemFilter");
+ tokenList = valueResult.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expecting the 'PorterStemFilter' to be applied on the index for the 'text' field", tokenList);
assertEquals("Expecting 4 tokens", 4, tokenList.size());
assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 4, 7, 1, null, false));
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java?rev=940459&r1=940458&r2=940459&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java Mon May 3 13:54:44 2010
@@ -173,7 +173,7 @@ public class FieldAnalysisRequestHandler
assertToken(tokenList.get(5), new TokenInfo("lazy", null, "<ALPHANUM>", 34, 38, 6, null, false));
assertToken(tokenList.get(6), new TokenInfo("brown", null, "<ALPHANUM>", 39, 44, 7, null, true));
assertToken(tokenList.get(7), new TokenInfo("dogs", null, "<ALPHANUM>", 45, 49, 8, null, false));
- tokenList = indexPart.get("org.apache.lucene.analysis.PorterStemFilter");
+ tokenList = indexPart.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expcting PorterStemFilter analysis breakdown", tokenList);
assertEquals(tokenList.size(), 8);
assertToken(tokenList.get(0), new TokenInfo("quick", null, "<ALPHANUM>", 4, 9, 1, null, false));
@@ -208,7 +208,7 @@ public class FieldAnalysisRequestHandler
assertEquals(2, tokenList.size());
assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, null, false));
assertToken(tokenList.get(1), new TokenInfo("brown", null, "<ALPHANUM>", 4, 9, 2, null, false));
- tokenList = queryPart.get("org.apache.lucene.analysis.PorterStemFilter");
+ tokenList = queryPart.get("org.apache.lucene.analysis.en.PorterStemFilter");
assertNotNull("Expcting PorterStemFilter analysis breakdown", tokenList);
assertEquals(2, tokenList.size());
assertToken(tokenList.get(0), new TokenInfo("fox", null, "<ALPHANUM>", 0, 3, 1, null, false));