You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/05/03 22:02:07 UTC
svn commit: r940591 - in /lucene/dev/trunk: lucene/
lucene/backwards/src/java/org/apache/lucene/analysis/
lucene/backwards/src/test/org/apache/lucene/analysis/
lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/
lucene/co...
Author: rmuir
Date: Mon May 3 20:02:06 2010
New Revision: 940591
URL: http://svn.apache.org/viewvc?rev=940591&view=rev
Log:
LUCENE-2413: consolidate ASCIIFolding and ISOLatin1Accent to contrib/analyzers
Added:
lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java
- copied, changed from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java
- copied, changed from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
- copied, changed from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java
- copied, changed from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
Removed:
lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java
lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java
lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon May 3 20:02:06 2010
@@ -6,6 +6,8 @@ Changes in backwards compatibility polic
* LUCENE-2413: Consolidated all Lucene analyzers into contrib/analyzers.
- o.a.l.analysis.PorterStemFilter -> o.a.l.analysis.en.PorterStemFilter
+ - o.a.l.analysis.ASCIIFoldingFilter -> o.a.l.analysis.miscellaneous.ASCIIFoldingFilter
+ - o.a.l.analysis.ISOLatin1AccentFilter -> o.a.l.analysis.miscellaneous.ISOLatin1AccentFilter
... (in progress)
* LUCENE-1458, LUCENE-2111, LUCENE-2354: Changes from flexible indexing:
Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java (from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java Mon May 3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -19,6 +19,8 @@ package org.apache.lucene.analysis;
import java.io.IOException;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java (from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java Mon May 3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +17,8 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java Mon May 3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.StringReader;
import java.util.List;
Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java (from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java Mon May 3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.StringReader;
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java Mon May 3 20:02:06 2010
@@ -17,14 +17,16 @@ package org.apache.lucene.queryParser.an
* limitations under the License.
*/
+import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.ASCIIFoldingFilter;
import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.util.LuceneTestCase;
@@ -105,6 +107,41 @@ public class TestAnalyzingQueryParser ex
}
+// TODO: Use a TestAnalyzer instead
+final class TestFoldingFilter extends TokenFilter {
+ final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ public TestFoldingFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ char term[] = termAtt.buffer();
+ for (int i = 0; i < term.length; i++)
+ switch(term[i]) {
+ case 'ü':
+ case 'Ã':
+ term[i] = 'u';
+ break;
+ case 'ö':
+ term[i] = 'o';
+ break;
+ case 'é':
+ term[i] = 'e';
+ break;
+ case 'ï':
+ term[i] = 'i';
+ break;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
+
final class ASCIIAnalyzer extends org.apache.lucene.analysis.Analyzer {
public ASCIIAnalyzer() {
}
@@ -113,7 +150,7 @@ final class ASCIIAnalyzer extends org.ap
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader);
result = new StandardFilter(result);
- result = new ASCIIFoldingFilter(result);
+ result = new TestFoldingFilter(result);
result = new LowerCaseFilter(LuceneTestCase.TEST_VERSION_CURRENT, result);
return result;
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java Mon May 3 20:02:06 2010
@@ -18,7 +18,7 @@
package org.apache.solr.analysis;
-import org.apache.lucene.analysis.ASCIIFoldingFilter;
+import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
import org.apache.lucene.analysis.TokenStream;
/** Factory for {@link ASCIIFoldingFilter} */
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java Mon May 3 20:02:06 2010
@@ -17,7 +17,7 @@
package org.apache.solr.analysis;
-import org.apache.lucene.analysis.ISOLatin1AccentFilter;
+import org.apache.lucene.analysis.miscellaneous.ISOLatin1AccentFilter;
import org.apache.lucene.analysis.TokenStream;
/** Factory for ISOLatin1AccentFilter