You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/01/10 13:41:47 UTC
svn commit: r1229520 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/ lucene/contrib/analyzers/ lucene/contrib/analyzers/common/
lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/contrib/analyzers/commo...
Author: simonw
Date: Tue Jan 10 12:41:46 2012
New Revision: 1229520
URL: http://svn.apache.org/viewvc?rev=1229520&view=rev
Log:
SOLR-3020: Add KeywordAttribute support to HunspellStemFilter
Added:
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
- copied, changed from r1229519, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/icu/ (props changed)
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt?rev=1229520&r1=1229519&r2=1229520&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt Tue Jan 10 12:41:46 2012
@@ -28,6 +28,9 @@ New Features
* LUCENE-3634: IndexReader's static main method was moved to a new
tool, CompoundFileExtractor, in contrib/misc. (Mike McCandless)
+* SOLR-3020: Add KeywordAttribute support to HunspellStemFilter. Terms marked as
+ keywords are not modified by the stemmer. (Simon Willnauer, Helge Jenssen)
+
API Changes
* LUCENE-3596: DirectoryTaxonomyWriter.openIndexWriter() now takes an
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java?rev=1229520&r1=1229519&r2=1229520&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemFilter.java Tue Jan 10 12:41:46 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.TokenF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.hunspell.HunspellStemmer.Stem;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
/**
@@ -34,6 +35,7 @@ public final class HunspellStemFilter ex
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
private final HunspellStemmer stemmer;
private List<Stem> buffer;
@@ -84,6 +86,10 @@ public final class HunspellStemFilter ex
return false;
}
+ if (keywordAtt.isKeyword()) {
+ return true;
+ }
+
buffer = dedup ? stemmer.uniqueStems(termAtt.buffer(), termAtt.length()) : stemmer.stem(termAtt.buffer(), termAtt.length());
if (buffer.isEmpty()) { // we do not know this word, return it unchanged
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (from r1229519, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java&r1=1229519&r2=1229520&rev=1229520&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java Tue Jan 10 12:41:46 2012
@@ -23,10 +23,9 @@ import java.text.ParseException;
import java.util.Arrays;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.KeywordMarkerFilter;
import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
-import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.util.Version;
import org.junit.BeforeClass;
public class HunspellStemFilterTest extends BaseTokenStreamTestCase {
@@ -40,7 +39,7 @@ public class HunspellStemFilterTest ext
InputStream affixStream = HunspellStemmerTest.class.getResourceAsStream("test.aff");
InputStream dictStream = HunspellStemmerTest.class.getResourceAsStream("test.dic");
- return new HunspellDictionary(affixStream, dictStream, Version.LUCENE_40, ignoreCase);
+ return new HunspellDictionary(affixStream, dictStream, TEST_VERSION_CURRENT, ignoreCase);
}
/**