You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/01/21 20:10:36 UTC
svn commit: r1234397 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/CHANGES.txt lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java

Author: uschindler
Date: Sat Jan 21 19:10:35 2012
New Revision: 1234397

URL: http://svn.apache.org/viewvc?rev=1234397&view=rev
Log:
Merged revision(s) 1234396 from lucene/dev/trunk:
LUCENE-3121: Add TypeTokenFilter that filters tokens based on their TypeAttribute

Added:
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java
      - copied, changed from r1234396, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java
      - copied, changed from r1234396, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/CHANGES.txt

Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1234397&r1=1234396&r2=1234397&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Sat Jan 21 19:10:35 2012
@@ -105,6 +105,9 @@ New Features
   input mapping to it) for FSTs that have strictly monotonic long
   outputs (such as an ord).  (Mike McCandless)
   
+* LUCENE-3121: Add TypeTokenFilter that filters tokens based on
+  their TypeAttribute.  (Tommaso Teofili via Uwe Schindler)
+  
 Bug fixes
 
 * LUCENE-3595: Fixed FieldCacheRangeFilter and FieldCacheTermsFilter

Copied: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java (from r1234396, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java?p2=lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java&r1=1234396&r2=1234397&rev=1234397&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java Sat Jan 21 19:10:35 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.core;
+package org.apache.lucene.analysis;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,9 +17,7 @@ package org.apache.lucene.analysis.core;
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.analysis.util.FilteringTokenFilter;
 
 import java.io.IOException;
 import java.util.Set;

Copied: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java (from r1234396, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java?p2=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java&r1=1234396&r2=1234397&rev=1234397&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java Sat Jan 21 19:10:35 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.core;
+package org.apache.lucene.analysis;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,8 +17,6 @@ package org.apache.lucene.analysis.core;
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -27,6 +25,7 @@ import org.apache.lucene.util.English;
 
 import java.io.IOException;
 import java.io.StringReader;
+import java.util.Collections;
 import java.util.Set;
 
 
@@ -34,7 +33,7 @@ public class TestTypeTokenFilter extends
 
   public void testTypeFilter() throws IOException {
     StringReader reader = new StringReader("121 is palindrome, while 123 is not");
-    Set<String> stopTypes = asSet("<NUM>");
+    Set<String> stopTypes = Collections.singleton("<NUM>");
     TokenStream stream = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
     assertTokenStreamContents(stream, new String[]{"is", "palindrome", "while", "is", "not"});
   }
@@ -53,8 +52,7 @@ public class TestTypeTokenFilter extends
       }
     }
     log(sb.toString());
-    String stopTypes[] = new String[]{"<NUM>"};
-    Set<String> stopSet = asSet(stopTypes);
+    Set<String> stopSet = Collections.singleton("<NUM>");
 
     // with increments
     StringReader reader = new StringReader(sb.toString());