You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/01/31 15:06:45 UTC
svn commit: r1065621 - in /lucene/dev/trunk: lucene/
modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/
modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/
Author: rmuir
Date: Mon Jan 31 14:06:45 2011
New Revision: 1065621
URL: http://svn.apache.org/viewvc?rev=1065621&view=rev
Log:
LUCENE-2901: fix consistency of KeywordMarkerFilter, it should only set, not unset the attribute
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1065621&r1=1065620&r2=1065621&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Jan 31 14:06:45 2011
@@ -747,8 +747,8 @@ New features
stopwords, and implement many analyzers in contrib with it.
(Simon Willnauer via Robert Muir)
-* LUCENE-2198: Support protected words in stemming TokenFilters using a
- new KeywordAttribute. (Simon Willnauer via Uwe Schindler)
+* LUCENE-2198, LUCENE-2901: Support protected words in stemming TokenFilters using a
+ new KeywordAttribute. (Simon Willnauer, Drew Farris via Uwe Schindler)
* LUCENE-2183, LUCENE-2240, LUCENE-2241: Added Unicode 4 support
to CharTokenizer and its subclasses. CharTokenizer now has new
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java?rev=1065621&r1=1065620&r2=1065621&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilter.java Mon Jan 31 14:06:45 2011
@@ -74,10 +74,12 @@ public final class KeywordMarkerFilter e
@Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- keywordAttr.setKeyword(keywordSet.contains(termAtt.buffer(), 0,
- termAtt.length()));
+ if (keywordSet.contains(termAtt.buffer(), 0, termAtt.length())) {
+ keywordAttr.setKeyword(true);
+ }
return true;
- } else
+ } else {
return false;
+ }
}
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java?rev=1065621&r1=1065620&r2=1065621&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java Mon Jan 31 14:06:45 2011
@@ -2,6 +2,7 @@ package org.apache.lucene.analysis.misce
import java.io.IOException;
import java.io.StringReader;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
@@ -57,6 +58,19 @@ public class TestKeywordMarkerFilter ext
"The quIck browN LuceneFox Jumps")), set2)), output);
}
+ // LUCENE-2901
+ public void testComposition() throws Exception {
+ TokenStream ts = new LowerCaseFilterMock(
+ new KeywordMarkerFilter(
+ new KeywordMarkerFilter(
+ new WhitespaceTokenizer(TEST_VERSION_CURRENT,
+ new StringReader("Dogs Trees Birds Houses")),
+ new HashSet<String>(Arrays.asList(new String[] { "Birds", "Houses" }))),
+ new HashSet<String>(Arrays.asList(new String[] { "Dogs", "Trees" }))));
+
+ assertTokenStreamContents(ts, new String[] { "Dogs", "Trees", "Birds", "Houses" });
+ }
+
public static final class LowerCaseFilterMock extends TokenFilter {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);