You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2012/02/03 10:14:09 UTC
svn commit: r1240035 - in /lucene/dev/branches/branch_3x/lucene/src:
java/org/apache/lucene/analysis/TypeTokenFilter.java
test/org/apache/lucene/analysis/TestTypeTokenFilter.java
Author: tommaso
Date: Fri Feb 3 09:14:08 2012
New Revision: 1240035
URL: http://svn.apache.org/viewvc?rev=1240035&view=rev
Log:
[LUCENE-3744] - applied patch for whiteList usage in TypeTokenFilter
Modified:
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java?rev=1240035&r1=1240034&r2=1240035&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/TypeTokenFilter.java Fri Feb 3 09:14:08 2012
@@ -29,17 +29,24 @@ public final class TypeTokenFilter exten
private final Set<String> stopTypes;
private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
+ private final boolean useWhiteList;
- public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes) {
+ public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
super(enablePositionIncrements, input);
this.stopTypes = stopTypes;
+ this.useWhiteList = useWhiteList;
+ }
+
+ public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes) {
+ this(enablePositionIncrements, input, stopTypes, false);
}
/**
- * Returns the next input Token whose typeAttribute.type() is not a stop type.
+ * By default accept the token if its type is not a stop type.
+ * When the useWhiteList parameter is set to true then accept the token if its type is contained in the stopTypes
*/
@Override
protected boolean accept() throws IOException {
- return !stopTypes.contains(typeAttribute.type());
+ return useWhiteList == stopTypes.contains(typeAttribute.type());
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java?rev=1240035&r1=1240034&r2=1240035&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestTypeTokenFilter.java Fri Feb 3 09:14:08 2012
@@ -23,9 +23,9 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.English;
+import java.util.Collections;
import java.io.IOException;
import java.io.StringReader;
-import java.util.Collections;
import java.util.Set;
@@ -81,6 +81,13 @@ public class TestTypeTokenFilter extends
stpf.close();
}
+ public void testTypeFilterWhitelist() throws IOException {
+ StringReader reader = new StringReader("121 is palindrome, while 123 is not");
+ Set<String> stopTypes = Collections.singleton("<NUM>");
+ TokenStream stream = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
+ assertTokenStreamContents(stream, new String[]{"121", "123"});
+ }
+
// print debug info depending on VERBOSE
private static void log(String s) {
if (VERBOSE) {
Re: svn commit: r1240035 - in /lucene/dev/branches/branch_3x/lucene/src:
java/org/apache/lucene/analysis/TypeTokenFilter.java test/org/apache/lucene/analysis/TestTypeTokenFilter.java
Posted by Tommaso Teofili <to...@gmail.com>.
Actually you just made the needed fixes, lesson learned, thanks again.
Tommaso
2012/2/7 Tommaso Teofili <to...@gmail.com>
> Thanks Uwe for noticing, I'll fix that.
> Tommaso
>
>
> 2012/2/4 Uwe Schindler <uw...@thetaphi.de>
>
>> One more thing:
>> Please merge changes from trunk to 3.x, not only apply patch twice. More
>> info about the sometimes complicated merging (because of move to modules of
>> some code parts): http://wiki.apache.org/lucene-java/SvnMerge
>>
>> I added the missing merge properties.
>>
>> -----
>> Uwe Schindler
>> H.-H.-Meier-Allee 63, D-28213 Bremen
>> http://www.thetaphi.de
>> eMail: uwe@thetaphi.de
>>
>>
>> > -----Original Message-----
>> > From: Uwe Schindler [mailto:uwe@thetaphi.de]
>> > Sent: Saturday, February 04, 2012 6:51 PM
>> > To: dev@lucene.apache.org
>> > Cc: tommaso.teofili@gmail.com
>> > Subject: RE: svn commit: r1240035 - in
>> > /lucene/dev/branches/branch_3x/lucene/src:
>> > java/org/apache/lucene/analysis/TypeTokenFilter.java
>> > test/org/apache/lucene/analysis/TestTypeTokenFilter.java
>> >
>> > Hi Tommaso,
>> >
>> > As you are a new committer, please take care of the following:
>> > - The branch 3.x of Lucene/Solr must still compile and test with Java
>> 5, so after
>> > merging from trunk, run and compile all tests with Java 5. There is a
>> > bug/feature/whatever in Java 6's compiler that it does not complain
>> about
>> > @Override on "-source 1.5 -target 1.5" when added to interface
>> > implementations (but it should, as @Override is not allowed there in
>> Java 5).
>> > - You had a merge relict ("x" somewhere).
>> >
>> > Uwe
>> >
>> > -----
>> > Uwe Schindler
>> > H.-H.-Meier-Allee 63, D-28213 Bremen
>> > http://www.thetaphi.de
>> > eMail: uwe@thetaphi.de
>> >
>> >
>> > > -----Original Message-----
>> > > From: tommaso@apache.org [mailto:tommaso@apache.org]
>> > > Sent: Friday, February 03, 2012 10:14 AM
>> > > To: commits@lucene.apache.org
>> > > Subject: svn commit: r1240035 - in
>> > > /lucene/dev/branches/branch_3x/lucene/src:
>> > > java/org/apache/lucene/analysis/TypeTokenFilter.java
>> > > test/org/apache/lucene/analysis/TestTypeTokenFilter.java
>> > >
>> > > Author: tommaso
>> > > Date: Fri Feb 3 09:14:08 2012
>> > > New Revision: 1240035
>> > >
>> > > URL: http://svn.apache.org/viewvc?rev=1240035&view=rev
>> > > Log:
>> > > [LUCENE-3744] - applied patch for whiteList usage in TypeTokenFilter
>> > >
>> > > Modified:
>> > >
>> > > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
>> > > is/T
>> > > ypeTokenFilter.java
>> > >
>> > > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
>> > > is/T
>> > > estTypeTokenFilter.java
>> > >
>> > > Modified:
>> > > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
>> > > is/T
>> > > ypeTokenFilter.java
>> > > URL:
>> > >
>> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/
>> > > java
>> > > /org/apache/lucene/analysis/TypeTokenFilter.java?rev=1240035&r1=124003
>> > > 4
>> > > &r2=1240035&view=diff
>> > >
>> > ================================================================
>> > > ==============
>> > > ---
>> > > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
>> > > is/T
>> > > ypeTokenFilter.java (original)
>> > > +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/an
>> > > +++ al ysis/TypeTokenFilter.java Fri Feb 3 09:14:08 2012
>> > > @@ -29,17 +29,24 @@ public final class TypeTokenFilter exten
>> > >
>> > > private final Set<String> stopTypes;
>> > > private final TypeAttribute typeAttribute =
>> > > addAttribute(TypeAttribute.class);
>> > > + private final boolean useWhiteList;
>> > >
>> > > - public TypeTokenFilter(boolean enablePositionIncrements,
>> > > TokenStream input, Set<String> stopTypes) {
>> > > + public TypeTokenFilter(boolean enablePositionIncrements,
>> > > + TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
>> > > super(enablePositionIncrements, input);
>> > > this.stopTypes = stopTypes;
>> > > + this.useWhiteList = useWhiteList; }
>> > > +
>> > > + public TypeTokenFilter(boolean enablePositionIncrements,
>> > > + TokenStream
>> > > input, Set<String> stopTypes) {
>> > > + this(enablePositionIncrements, input, stopTypes, false);
>> > > }
>> > >
>> > > /**
>> > > - * Returns the next input Token whose typeAttribute.type() is not
>> a stop
>> > type.
>> > > + * By default accept the token if its type is not a stop type.
>> > > + * When the useWhiteList parameter is set to true then accept the
>> > > + token if its type is contained in the stopTypes
>> > > */
>> > > @Override
>> > > protected boolean accept() throws IOException {
>> > > - return !stopTypes.contains(typeAttribute.type());
>> > > + return useWhiteList == stopTypes.contains(typeAttribute.type());
>> > > }
>> > > }
>> > >
>> > > Modified:
>> > > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
>> > > is/T
>> > > estTypeTokenFilter.java
>> > > URL:
>> > >
>> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/
>> > > test/
>> > > org/apache/lucene/analysis/TestTypeTokenFilter.java?rev=1240035&r1=124
>> > > 00
>> > > 34&r2=1240035&view=diff
>> > >
>> > ================================================================
>> > > ==============
>> > > ---
>> > > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
>> > > is/T
>> > > estTypeTokenFilter.java (original)
>> > > +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/an
>> > > +++ al ysis/TestTypeTokenFilter.java Fri Feb 3 09:14:08 2012
>> > > @@ -23,9 +23,9 @@ import org.apache.lucene.analysis.tokena import
>> > > org.apache.lucene.analysis.tokenattributes.TypeAttribute;
>> > > import org.apache.lucene.util.English;
>> > >
>> > > +import java.util.Collections;
>> > > import java.io.IOException;
>> > > import java.io.StringReader;
>> > > -import java.util.Collections;
>> > > import java.util.Set;
>> > >
>> > >
>> > > @@ -81,6 +81,13 @@ public class TestTypeTokenFilter extends
>> > > stpf.close();
>> > > }
>> > >
>> > > + public void testTypeFilterWhitelist() throws IOException {
>> > > + StringReader reader = new StringReader("121 is palindrome, while
>> > > + 123 is
>> > > not");
>> > > + Set<String> stopTypes = Collections.singleton("<NUM>");
>> > > + TokenStream stream = new TypeTokenFilter(true, new
>> > > StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
>> > > + assertTokenStreamContents(stream, new String[]{"121", "123"});
>> > > + }
>> > > +
>> > > // print debug info depending on VERBOSE
>> > > private static void log(String s) {
>> > > if (VERBOSE) {
>> >
>> >
>> >
>> > ---------------------------------------------------------------------
>> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For
>> additional
>> > commands, e-mail: dev-help@lucene.apache.org
>>
>>
>
Re: svn commit: r1240035 - in /lucene/dev/branches/branch_3x/lucene/src:
java/org/apache/lucene/analysis/TypeTokenFilter.java test/org/apache/lucene/analysis/TestTypeTokenFilter.java
Posted by Tommaso Teofili <to...@gmail.com>.
Thanks Uwe for noticing, I'll fix that.
Tommaso
2012/2/4 Uwe Schindler <uw...@thetaphi.de>
> One more thing:
> Please merge changes from trunk to 3.x, not only apply patch twice. More
> info about the sometimes complicated merging (because of move to modules of
> some code parts): http://wiki.apache.org/lucene-java/SvnMerge
>
> I added the missing merge properties.
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
>
> > -----Original Message-----
> > From: Uwe Schindler [mailto:uwe@thetaphi.de]
> > Sent: Saturday, February 04, 2012 6:51 PM
> > To: dev@lucene.apache.org
> > Cc: tommaso.teofili@gmail.com
> > Subject: RE: svn commit: r1240035 - in
> > /lucene/dev/branches/branch_3x/lucene/src:
> > java/org/apache/lucene/analysis/TypeTokenFilter.java
> > test/org/apache/lucene/analysis/TestTypeTokenFilter.java
> >
> > Hi Tommaso,
> >
> > As you are a new committer, please take care of the following:
> > - The branch 3.x of Lucene/Solr must still compile and test with Java 5,
> so after
> > merging from trunk, run and compile all tests with Java 5. There is a
> > bug/feature/whatever in Java 6's compiler that it does not complain about
> > @Override on "-source 1.5 -target 1.5" when added to interface
> > implementations (but it should, as @Override is not allowed there in
> Java 5).
> > - You had a merge relict ("x" somewhere).
> >
> > Uwe
> >
> > -----
> > Uwe Schindler
> > H.-H.-Meier-Allee 63, D-28213 Bremen
> > http://www.thetaphi.de
> > eMail: uwe@thetaphi.de
> >
> >
> > > -----Original Message-----
> > > From: tommaso@apache.org [mailto:tommaso@apache.org]
> > > Sent: Friday, February 03, 2012 10:14 AM
> > > To: commits@lucene.apache.org
> > > Subject: svn commit: r1240035 - in
> > > /lucene/dev/branches/branch_3x/lucene/src:
> > > java/org/apache/lucene/analysis/TypeTokenFilter.java
> > > test/org/apache/lucene/analysis/TestTypeTokenFilter.java
> > >
> > > Author: tommaso
> > > Date: Fri Feb 3 09:14:08 2012
> > > New Revision: 1240035
> > >
> > > URL: http://svn.apache.org/viewvc?rev=1240035&view=rev
> > > Log:
> > > [LUCENE-3744] - applied patch for whiteList usage in TypeTokenFilter
> > >
> > > Modified:
> > >
> > > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
> > > is/T
> > > ypeTokenFilter.java
> > >
> > > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
> > > is/T
> > > estTypeTokenFilter.java
> > >
> > > Modified:
> > > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
> > > is/T
> > > ypeTokenFilter.java
> > > URL:
> > > http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/
> > > java
> > > /org/apache/lucene/analysis/TypeTokenFilter.java?rev=1240035&r1=124003
> > > 4
> > > &r2=1240035&view=diff
> > >
> > ================================================================
> > > ==============
> > > ---
> > > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
> > > is/T
> > > ypeTokenFilter.java (original)
> > > +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/an
> > > +++ al ysis/TypeTokenFilter.java Fri Feb 3 09:14:08 2012
> > > @@ -29,17 +29,24 @@ public final class TypeTokenFilter exten
> > >
> > > private final Set<String> stopTypes;
> > > private final TypeAttribute typeAttribute =
> > > addAttribute(TypeAttribute.class);
> > > + private final boolean useWhiteList;
> > >
> > > - public TypeTokenFilter(boolean enablePositionIncrements,
> > > TokenStream input, Set<String> stopTypes) {
> > > + public TypeTokenFilter(boolean enablePositionIncrements,
> > > + TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
> > > super(enablePositionIncrements, input);
> > > this.stopTypes = stopTypes;
> > > + this.useWhiteList = useWhiteList; }
> > > +
> > > + public TypeTokenFilter(boolean enablePositionIncrements,
> > > + TokenStream
> > > input, Set<String> stopTypes) {
> > > + this(enablePositionIncrements, input, stopTypes, false);
> > > }
> > >
> > > /**
> > > - * Returns the next input Token whose typeAttribute.type() is not a
> stop
> > type.
> > > + * By default accept the token if its type is not a stop type.
> > > + * When the useWhiteList parameter is set to true then accept the
> > > + token if its type is contained in the stopTypes
> > > */
> > > @Override
> > > protected boolean accept() throws IOException {
> > > - return !stopTypes.contains(typeAttribute.type());
> > > + return useWhiteList == stopTypes.contains(typeAttribute.type());
> > > }
> > > }
> > >
> > > Modified:
> > > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
> > > is/T
> > > estTypeTokenFilter.java
> > > URL:
> > > http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/
> > > test/
> > > org/apache/lucene/analysis/TestTypeTokenFilter.java?rev=1240035&r1=124
> > > 00
> > > 34&r2=1240035&view=diff
> > >
> > ================================================================
> > > ==============
> > > ---
> > > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
> > > is/T
> > > estTypeTokenFilter.java (original)
> > > +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/an
> > > +++ al ysis/TestTypeTokenFilter.java Fri Feb 3 09:14:08 2012
> > > @@ -23,9 +23,9 @@ import org.apache.lucene.analysis.tokena import
> > > org.apache.lucene.analysis.tokenattributes.TypeAttribute;
> > > import org.apache.lucene.util.English;
> > >
> > > +import java.util.Collections;
> > > import java.io.IOException;
> > > import java.io.StringReader;
> > > -import java.util.Collections;
> > > import java.util.Set;
> > >
> > >
> > > @@ -81,6 +81,13 @@ public class TestTypeTokenFilter extends
> > > stpf.close();
> > > }
> > >
> > > + public void testTypeFilterWhitelist() throws IOException {
> > > + StringReader reader = new StringReader("121 is palindrome, while
> > > + 123 is
> > > not");
> > > + Set<String> stopTypes = Collections.singleton("<NUM>");
> > > + TokenStream stream = new TypeTokenFilter(true, new
> > > StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
> > > + assertTokenStreamContents(stream, new String[]{"121", "123"});
> > > + }
> > > +
> > > // print debug info depending on VERBOSE
> > > private static void log(String s) {
> > > if (VERBOSE) {
> >
> >
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For additional
> > commands, e-mail: dev-help@lucene.apache.org
>
>
RE: svn commit: r1240035 - in /lucene/dev/branches/branch_3x/lucene/src: java/org/apache/lucene/analysis/TypeTokenFilter.java test/org/apache/lucene/analysis/TestTypeTokenFilter.java
Posted by Uwe Schindler <uw...@thetaphi.de>.
One more thing:
Please merge changes from trunk to 3.x, not only apply patch twice. More info about the sometimes complicated merging (because of move to modules of some code parts): http://wiki.apache.org/lucene-java/SvnMerge
I added the missing merge properties.
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Uwe Schindler [mailto:uwe@thetaphi.de]
> Sent: Saturday, February 04, 2012 6:51 PM
> To: dev@lucene.apache.org
> Cc: tommaso.teofili@gmail.com
> Subject: RE: svn commit: r1240035 - in
> /lucene/dev/branches/branch_3x/lucene/src:
> java/org/apache/lucene/analysis/TypeTokenFilter.java
> test/org/apache/lucene/analysis/TestTypeTokenFilter.java
>
> Hi Tommaso,
>
> As you are a new committer, please take care of the following:
> - The branch 3.x of Lucene/Solr must still compile and test with Java 5, so after
> merging from trunk, run and compile all tests with Java 5. There is a
> bug/feature/whatever in Java 6's compiler that it does not complain about
> @Override on "-source 1.5 -target 1.5" when added to interface
> implementations (but it should, as @Override is not allowed there in Java 5).
> - You had a merge relict ("x" somewhere).
>
> Uwe
>
> -----
> Uwe Schindler
> H.-H.-Meier-Allee 63, D-28213 Bremen
> http://www.thetaphi.de
> eMail: uwe@thetaphi.de
>
>
> > -----Original Message-----
> > From: tommaso@apache.org [mailto:tommaso@apache.org]
> > Sent: Friday, February 03, 2012 10:14 AM
> > To: commits@lucene.apache.org
> > Subject: svn commit: r1240035 - in
> > /lucene/dev/branches/branch_3x/lucene/src:
> > java/org/apache/lucene/analysis/TypeTokenFilter.java
> > test/org/apache/lucene/analysis/TestTypeTokenFilter.java
> >
> > Author: tommaso
> > Date: Fri Feb 3 09:14:08 2012
> > New Revision: 1240035
> >
> > URL: http://svn.apache.org/viewvc?rev=1240035&view=rev
> > Log:
> > [LUCENE-3744] - applied patch for whiteList usage in TypeTokenFilter
> >
> > Modified:
> >
> > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
> > is/T
> > ypeTokenFilter.java
> >
> > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
> > is/T
> > estTypeTokenFilter.java
> >
> > Modified:
> > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
> > is/T
> > ypeTokenFilter.java
> > URL:
> > http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/
> > java
> > /org/apache/lucene/analysis/TypeTokenFilter.java?rev=1240035&r1=124003
> > 4
> > &r2=1240035&view=diff
> >
> ================================================================
> > ==============
> > ---
> > lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analys
> > is/T
> > ypeTokenFilter.java (original)
> > +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/an
> > +++ al ysis/TypeTokenFilter.java Fri Feb 3 09:14:08 2012
> > @@ -29,17 +29,24 @@ public final class TypeTokenFilter exten
> >
> > private final Set<String> stopTypes;
> > private final TypeAttribute typeAttribute =
> > addAttribute(TypeAttribute.class);
> > + private final boolean useWhiteList;
> >
> > - public TypeTokenFilter(boolean enablePositionIncrements,
> > TokenStream input, Set<String> stopTypes) {
> > + public TypeTokenFilter(boolean enablePositionIncrements,
> > + TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
> > super(enablePositionIncrements, input);
> > this.stopTypes = stopTypes;
> > + this.useWhiteList = useWhiteList; }
> > +
> > + public TypeTokenFilter(boolean enablePositionIncrements,
> > + TokenStream
> > input, Set<String> stopTypes) {
> > + this(enablePositionIncrements, input, stopTypes, false);
> > }
> >
> > /**
> > - * Returns the next input Token whose typeAttribute.type() is not a stop
> type.
> > + * By default accept the token if its type is not a stop type.
> > + * When the useWhiteList parameter is set to true then accept the
> > + token if its type is contained in the stopTypes
> > */
> > @Override
> > protected boolean accept() throws IOException {
> > - return !stopTypes.contains(typeAttribute.type());
> > + return useWhiteList == stopTypes.contains(typeAttribute.type());
> > }
> > }
> >
> > Modified:
> > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
> > is/T
> > estTypeTokenFilter.java
> > URL:
> > http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/
> > test/
> > org/apache/lucene/analysis/TestTypeTokenFilter.java?rev=1240035&r1=124
> > 00
> > 34&r2=1240035&view=diff
> >
> ================================================================
> > ==============
> > ---
> > lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analys
> > is/T
> > estTypeTokenFilter.java (original)
> > +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/an
> > +++ al ysis/TestTypeTokenFilter.java Fri Feb 3 09:14:08 2012
> > @@ -23,9 +23,9 @@ import org.apache.lucene.analysis.tokena import
> > org.apache.lucene.analysis.tokenattributes.TypeAttribute;
> > import org.apache.lucene.util.English;
> >
> > +import java.util.Collections;
> > import java.io.IOException;
> > import java.io.StringReader;
> > -import java.util.Collections;
> > import java.util.Set;
> >
> >
> > @@ -81,6 +81,13 @@ public class TestTypeTokenFilter extends
> > stpf.close();
> > }
> >
> > + public void testTypeFilterWhitelist() throws IOException {
> > + StringReader reader = new StringReader("121 is palindrome, while
> > + 123 is
> > not");
> > + Set<String> stopTypes = Collections.singleton("<NUM>");
> > + TokenStream stream = new TypeTokenFilter(true, new
> > StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
> > + assertTokenStreamContents(stream, new String[]{"121", "123"});
> > + }
> > +
> > // print debug info depending on VERBOSE
> > private static void log(String s) {
> > if (VERBOSE) {
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org For additional
> commands, e-mail: dev-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
RE: svn commit: r1240035 - in /lucene/dev/branches/branch_3x/lucene/src: java/org/apache/lucene/analysis/TypeTokenFilter.java test/org/apache/lucene/analysis/TestTypeTokenFilter.java
Posted by Uwe Schindler <uw...@thetaphi.de>.
Hi Tommaso,
As you are a new committer, please take care of the following:
- The branch 3.x of Lucene/Solr must still compile and test with Java 5, so after merging from trunk, run and compile all tests with Java 5. There is a bug/feature/whatever in Java 6's compiler that it does not complain about @Override on "-source 1.5 -target 1.5" when added to interface implementations (but it should, as @Override is not allowed there in Java 5).
- You had a merge relict ("x" somewhere).
Uwe
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: tommaso@apache.org [mailto:tommaso@apache.org]
> Sent: Friday, February 03, 2012 10:14 AM
> To: commits@lucene.apache.org
> Subject: svn commit: r1240035 - in
> /lucene/dev/branches/branch_3x/lucene/src:
> java/org/apache/lucene/analysis/TypeTokenFilter.java
> test/org/apache/lucene/analysis/TestTypeTokenFilter.java
>
> Author: tommaso
> Date: Fri Feb 3 09:14:08 2012
> New Revision: 1240035
>
> URL: http://svn.apache.org/viewvc?rev=1240035&view=rev
> Log:
> [LUCENE-3744] - applied patch for whiteList usage in TypeTokenFilter
>
> Modified:
>
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/T
> ypeTokenFilter.java
>
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/T
> estTypeTokenFilter.java
>
> Modified:
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/T
> ypeTokenFilter.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java
> /org/apache/lucene/analysis/TypeTokenFilter.java?rev=1240035&r1=1240034
> &r2=1240035&view=diff
> ================================================================
> ==============
> ---
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/T
> ypeTokenFilter.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/anal
> +++ ysis/TypeTokenFilter.java Fri Feb 3 09:14:08 2012
> @@ -29,17 +29,24 @@ public final class TypeTokenFilter exten
>
> private final Set<String> stopTypes;
> private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
> + private final boolean useWhiteList;
>
> - public TypeTokenFilter(boolean enablePositionIncrements, TokenStream
> input, Set<String> stopTypes) {
> + public TypeTokenFilter(boolean enablePositionIncrements, TokenStream
> + input, Set<String> stopTypes, boolean useWhiteList) {
> super(enablePositionIncrements, input);
> this.stopTypes = stopTypes;
> + this.useWhiteList = useWhiteList;
> + }
> +
> + public TypeTokenFilter(boolean enablePositionIncrements, TokenStream
> input, Set<String> stopTypes) {
> + this(enablePositionIncrements, input, stopTypes, false);
> }
>
> /**
> - * Returns the next input Token whose typeAttribute.type() is not a stop type.
> + * By default accept the token if its type is not a stop type.
> + * When the useWhiteList parameter is set to true then accept the
> + token if its type is contained in the stopTypes
> */
> @Override
> protected boolean accept() throws IOException {
> - return !stopTypes.contains(typeAttribute.type());
> + return useWhiteList == stopTypes.contains(typeAttribute.type());
> }
> }
>
> Modified:
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/T
> estTypeTokenFilter.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/
> org/apache/lucene/analysis/TestTypeTokenFilter.java?rev=1240035&r1=12400
> 34&r2=1240035&view=diff
> ================================================================
> ==============
> ---
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/T
> estTypeTokenFilter.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/anal
> +++ ysis/TestTypeTokenFilter.java Fri Feb 3 09:14:08 2012
> @@ -23,9 +23,9 @@ import org.apache.lucene.analysis.tokena import
> org.apache.lucene.analysis.tokenattributes.TypeAttribute;
> import org.apache.lucene.util.English;
>
> +import java.util.Collections;
> import java.io.IOException;
> import java.io.StringReader;
> -import java.util.Collections;
> import java.util.Set;
>
>
> @@ -81,6 +81,13 @@ public class TestTypeTokenFilter extends
> stpf.close();
> }
>
> + public void testTypeFilterWhitelist() throws IOException {
> + StringReader reader = new StringReader("121 is palindrome, while 123 is
> not");
> + Set<String> stopTypes = Collections.singleton("<NUM>");
> + TokenStream stream = new TypeTokenFilter(true, new
> StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
> + assertTokenStreamContents(stream, new String[]{"121", "123"});
> + }
> +
> // print debug info depending on VERBOSE
> private static void log(String s) {
> if (VERBOSE) {
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org