You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/05/03 22:02:07 UTC

svn commit: r940591 - in /lucene/dev/trunk: lucene/ lucene/backwards/src/java/org/apache/lucene/analysis/ lucene/backwards/src/test/org/apache/lucene/analysis/ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ lucene/co...

Author: rmuir
Date: Mon May  3 20:02:06 2010
New Revision: 940591

URL: http://svn.apache.org/viewvc?rev=940591&view=rev
Log:
LUCENE-2413: consolidate ASCIIFolding and ISOLatin1Accent to contrib/analyzers

Added:
    lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java
      - copied, changed from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java
      - copied, changed from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
    lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
      - copied, changed from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java
      - copied, changed from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
Removed:
    lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/backwards/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
    lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java
    lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon May  3 20:02:06 2010
@@ -6,6 +6,8 @@ Changes in backwards compatibility polic
 
 * LUCENE-2413: Consolidated all Lucene analyzers into contrib/analyzers.
   - o.a.l.analysis.PorterStemFilter -> o.a.l.analysis.en.PorterStemFilter
+  - o.a.l.analysis.ASCIIFoldingFilter -> o.a.l.analysis.miscellaneous.ASCIIFoldingFilter
+  - o.a.l.analysis.ISOLatin1AccentFilter -> o.a.l.analysis.miscellaneous.ISOLatin1AccentFilter
   ... (in progress)
 
 * LUCENE-1458, LUCENE-2111, LUCENE-2354: Changes from flexible indexing:

Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java (from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java Mon May  3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -19,6 +19,8 @@ package org.apache.lucene.analysis;
 
 import java.io.IOException;
 
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.RamUsageEstimator;

Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java (from r940564, lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ISOLatin1AccentFilter.java Mon May  3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +17,8 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**

Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java Mon May  3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import java.io.StringReader;
 import java.util.List;

Copied: lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java (from r940564, lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java?p2=lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java&r1=940564&r2=940591&rev=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestISOLatin1AccentFilter.java Mon May  3 20:02:06 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis;
+package org.apache.lucene.analysis.miscellaneous;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +17,9 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import java.io.StringReader;
 

Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java Mon May  3 20:02:06 2010
@@ -17,14 +17,16 @@ package org.apache.lucene.queryParser.an
  * limitations under the License.
  */
 
+import java.io.IOException;
 import java.io.Reader;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.ASCIIFoldingFilter;
 import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -105,6 +107,41 @@ public class TestAnalyzingQueryParser ex
 
 }
 
+// TODO: Use a TestAnalyzer instead
+final class TestFoldingFilter extends TokenFilter {
+  final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+  public TestFoldingFilter(TokenStream input) {
+    super(input);
+  }
+
+  @Override
+  public boolean incrementToken() throws IOException {
+    if (input.incrementToken()) {
+      char term[] = termAtt.buffer();
+      for (int i = 0; i < term.length; i++)
+        switch(term[i]) {
+          case 'ü':
+          case 'Ü':
+            term[i] = 'u'; 
+            break;
+          case 'ö': 
+            term[i] = 'o'; 
+            break;
+          case 'é': 
+            term[i] = 'e'; 
+            break;
+          case 'ï': 
+            term[i] = 'i'; 
+            break;
+        }
+      return true;
+    } else {
+      return false;
+    }
+  }
+}
+
 final class ASCIIAnalyzer extends org.apache.lucene.analysis.Analyzer {
   public ASCIIAnalyzer() {
   }
@@ -113,7 +150,7 @@ final class ASCIIAnalyzer extends org.ap
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new StandardTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader);
     result = new StandardFilter(result);
-    result = new ASCIIFoldingFilter(result);
+    result = new TestFoldingFilter(result);
     result = new LowerCaseFilter(LuceneTestCase.TEST_VERSION_CURRENT, result);
     return result;
   }

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ASCIIFoldingFilterFactory.java Mon May  3 20:02:06 2010
@@ -18,7 +18,7 @@
 
 
 package org.apache.solr.analysis;
-import org.apache.lucene.analysis.ASCIIFoldingFilter;
+import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
 import org.apache.lucene.analysis.TokenStream;
 
 /** Factory for {@link ASCIIFoldingFilter} */

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java?rev=940591&r1=940590&r2=940591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/ISOLatin1AccentFilterFactory.java Mon May  3 20:02:06 2010
@@ -17,7 +17,7 @@
  
 package org.apache.solr.analysis;
 
-import org.apache.lucene.analysis.ISOLatin1AccentFilter;
+import org.apache.lucene.analysis.miscellaneous.ISOLatin1AccentFilter;
 import org.apache.lucene.analysis.TokenStream;
 
 /** Factory for ISOLatin1AccentFilter