You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/07/14 14:48:57 UTC
svn commit: r964034 [2/3] - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/ lucene/backwards/src/test/org/apache/lucene/analysis/
lucene/backwards/src/test/org/apache/lucene/document/
lucene/backwards/src/test/org/apache/lucene/inde...
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/id/IndonesianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/id/IndonesianStemmer.java?rev=964034&r1=964033&r2=964034&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/id/IndonesianStemmer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/id/IndonesianStemmer.java Wed Jul 14 12:48:54 2010
@@ -17,6 +17,8 @@ package org.apache.lucene.analysis.id;
* limitations under the License.
*/
+import static org.apache.lucene.analysis.util.StemmerUtil.*;
+
/**
* Stemmer for Indonesian.
* <p>
@@ -266,39 +268,5 @@ public class IndonesianStemmer {
return length - 1;
}
return length;
- }
-
- private boolean startsWith(char s[], int len, String prefix) {
- final int prefixLen = prefix.length();
- if (prefixLen > len)
- return false;
- for (int i = 0; i < prefixLen; i++)
- if (s[i] != prefix.charAt(i))
- return false;
- return true;
- }
-
- private boolean endsWith(char s[], int len, String suffix) {
- final int suffixLen = suffix.length();
- if (suffixLen > len)
- return false;
- for (int i = suffixLen - 1; i >= 0; i--)
- if (s[len -(suffixLen - i)] != suffix.charAt(i))
- return false;
-
- return true;
- }
-
- private int deleteN(char s[], int pos, int len, int nChars) {
- for (int i = 0; i < nChars; i++)
- len = delete(s, pos, len);
- return len;
- }
-
- private int delete(char s[], int pos, int len) {
- if (pos < len)
- System.arraycopy(s, pos + 1, s, pos, len - pos - 1);
-
- return len - 1;
- }
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java?rev=964034&r1=964033&r2=964034&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java Wed Jul 14 12:48:54 2010
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.in;
import java.util.BitSet;
import java.util.IdentityHashMap;
import static java.lang.Character.UnicodeBlock.*;
+import static org.apache.lucene.analysis.util.StemmerUtil.*;
/**
* Normalizes the Unicode representation of text in Indian languages.
@@ -290,14 +291,4 @@ public class IndicNormalizer {
return len;
}
-
- /**
- * Delete a character in-place
- */
- private int delete(char s[], int pos, int len) {
- if (pos < len)
- System.arraycopy(s, pos + 1, s, pos, len - pos - 1);
-
- return len - 1;
- }
}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.KeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.KeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemFilter.java Wed Jul 14 12:48:54 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.KeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemmer.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemmer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemmer.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemmer.java&p1=lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemmer.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemmer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/pt/PortugueseMinimalStemmer.java Wed Jul 14 12:48:54 2010
@@ -2,7 +2,7 @@ package org.apache.lucene.analysis.pt;
import java.util.Arrays;
-import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.util.Version;
/**
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.KeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.KeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
Propchange: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,5 +1,5 @@
-/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
-/lucene/dev/trunk/modules/analysis/common/src/test:942235,945090,946139,950008,950042
+/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
+/lucene/dev/trunk/modules/analysis/common/src/test:942235,945090,946139,950008,950042,964019
/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test:924791,924850,930201
/lucene/java/branches/lucene_2_4/contrib/analyzers/common/src/test:748824
/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test:817269-818600,825998,829134,829816,829881,831036,896850,909334,948516
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link GermanLightStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link GermanMinimalStemFilter}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=964034&r1=964033&r2=964034&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Wed Jul 14 12:48:54 2010
@@ -17,17 +17,17 @@ package org.apache.lucene.analysis.de;
* limitations under the License.
*/
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
-import java.io.StringReader;
+import java.io.InputStream;
+import java.io.Reader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.KeywordTokenizer;
import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Test the German stemmer. The stemming algorithm is known to work less
@@ -38,25 +38,18 @@ import org.apache.lucene.analysis.Tokeni
public class TestGermanStemFilter extends BaseTokenStreamTestCase {
public void testStemming() throws Exception {
- Tokenizer tokenizer = new KeywordTokenizer(new StringReader(""));
- TokenFilter filter = new GermanStemFilter(new LowerCaseFilter(TEST_VERSION_CURRENT, tokenizer));
- // read test cases from external file:
- InputStreamReader isr = new InputStreamReader(getClass().getResourceAsStream("data.txt"), "iso-8859-1");
- BufferedReader breader = new BufferedReader(isr);
- while(true) {
- String line = breader.readLine();
- if (line == null)
- break;
- line = line.trim();
- if (line.startsWith("#") || line.equals(""))
- continue; // ignore comments and empty lines
- String[] parts = line.split(";");
- //System.out.println(parts[0] + " -- " + parts[1]);
- tokenizer.reset(new StringReader(parts[0]));
- filter.reset();
- assertTokenStreamContents(filter, new String[] { parts[1] });
- }
- breader.close();
- isr.close();
+ Analyzer analyzer = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ Tokenizer t = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(t,
+ new GermanStemFilter(new LowerCaseFilter(TEST_VERSION_CURRENT, t)));
+ }
+ };
+
+ InputStream vocOut = getClass().getResourceAsStream("data.txt");
+ assertVocabulary(analyzer, vocOut);
+ vocOut.close();
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/data.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/data.txt?rev=964034&r1=964033&r2=964034&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/data.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/data.txt Wed Jul 14 12:48:54 2010
@@ -1,48 +1,48 @@
# German special characters are replaced:
-häufig;haufig
+häufig haufig
# here the stemmer works okay, it maps related words to the same stem:
-abschließen;abschliess
-abschließender;abschliess
-abschließendes;abschliess
-abschließenden;abschliess
-
-Tisch;tisch
-Tische;tisch
-Tischen;tisch
-
-Haus;hau
-Hauses;hau
-Häuser;hau
-Häusern;hau
+abschlieÃen abschliess
+abschlieÃender abschliess
+abschlieÃendes abschliess
+abschlieÃenden abschliess
+
+Tisch tisch
+Tische tisch
+Tischen tisch
+
+Haus hau
+Hauses hau
+Häuser hau
+Häusern hau
# here's a case where overstemming occurs, i.e. a word is
# mapped to the same stem as unrelated words:
-hauen;hau
+hauen hau
# here's a case where understemming occurs, i.e. two related words
# are not mapped to the same stem. This is the case with basically
# all irregular forms:
-Drama;drama
-Dramen;dram
+Drama drama
+Dramen dram
-# replace "ß" with 'ss':
-Ausmaß;ausmass
+# replace "Ã" with 'ss':
+Ausmaà ausmass
# fake words to test if suffixes are cut off:
-xxxxxe;xxxxx
-xxxxxs;xxxxx
-xxxxxn;xxxxx
-xxxxxt;xxxxx
-xxxxxem;xxxxx
-xxxxxer;xxxxx
-xxxxxnd;xxxxx
+xxxxxe xxxxx
+xxxxxs xxxxx
+xxxxxn xxxxx
+xxxxxt xxxxx
+xxxxxem xxxxx
+xxxxxer xxxxx
+xxxxxnd xxxxx
# the suffixes are also removed when combined:
-xxxxxetende;xxxxx
+xxxxxetende xxxxx
# words that are shorter than four charcters are not changed:
-xxe;xxe
+xxe xxe
# -em and -er are not removed from words shorter than five characters:
-xxem;xxem
-xxer;xxer
+xxem xxem
+xxer xxer
# -nd is not removed from words shorter than six characters:
-xxxnd;xxxnd
+xxxnd xxxnd
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,8 +23,8 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
/**
* Simple tests for {@link EnglishMinimalStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link SpanishLightStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link FinnishLightStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link FrenchLightStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link FrenchMinimalStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link HungarianLightStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link ItalianLightStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -24,11 +24,11 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link PortugueseLightStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java Wed Jul 14 12:48:54 2010
@@ -24,11 +24,11 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link PortugueseMinimalStemFilter}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link RussianLightStemFilter}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java?rev=964034&r1=964033&r2=964034&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java Wed Jul 14 12:48:54 2010
@@ -17,71 +17,35 @@ package org.apache.lucene.analysis.ru;
* limitations under the License.
*/
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.KeywordTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.LuceneTestCase;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.InputStreamReader;
-import java.io.FileInputStream;
-import java.util.ArrayList;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* @deprecated Remove this test class (and its datafiles!) in Lucene 4.0
*/
@Deprecated
-public class TestRussianStem extends LuceneTestCase
-{
- private ArrayList<String> words = new ArrayList<String>();
- private ArrayList<String> stems = new ArrayList<String>();
-
- public TestRussianStem(String name)
- {
- super(name);
- }
-
- /**
- * @see TestCase#setUp()
- */
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- //System.out.println(new java.util.Date());
- String str;
-
- // open and read words into an array list
- BufferedReader inWords =
- new BufferedReader(
- new InputStreamReader(
- getClass().getResourceAsStream("wordsUTF8.txt"),
- "UTF-8"));
- while ((str = inWords.readLine()) != null)
- {
- words.add(str);
- }
- inWords.close();
-
- // open and read stems into an array list
- BufferedReader inStems =
- new BufferedReader(
- new InputStreamReader(
- getClass().getResourceAsStream("stemsUTF8.txt"),
- "UTF-8"));
- while ((str = inStems.readLine()) != null)
- {
- stems.add(str);
- }
- inStems.close();
- }
-
- public void testStem()
- {
- for (int i = 0; i < words.size(); i++)
- {
- //if ( (i % 100) == 0 ) System.err.println(i);
- String realStem =
- RussianStemmer.stemWord(
- words.get(i));
- assertEquals("unicode", stems.get(i), realStem);
- }
- }
-
+public class TestRussianStem extends LuceneTestCase {
+ public void testStem() throws IOException {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ Tokenizer t = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(t, new RussianStemFilter(t));
+ }
+ };
+ InputStream voc = getClass().getResourceAsStream("wordsUTF8.txt");
+ InputStream out = getClass().getResourceAsStream("stemsUTF8.txt");
+ assertVocabulary(a, voc, out);
+ voc.close();
+ out.close();
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java?rev=964034&r1=964033&r2=964034&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java Wed Jul 14 12:48:54 2010
@@ -17,38 +17,21 @@ package org.apache.lucene.analysis.snowb
* limitations under the License.
*/
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.StringReader;
-import java.util.zip.ZipFile;
+import java.io.Reader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.KeywordTokenizer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.util.LuceneTestCase;
+
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Test the snowball filters against the snowball data tests
*/
-public class TestSnowballVocab extends BaseTokenStreamTestCase {
- private Tokenizer tokenizer = new KeywordTokenizer(new StringReader(""));
- ZipFile zipFile = null;
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- this.zipFile = new ZipFile(getDataFile("TestSnowballVocabData.zip"));
- }
-
- @Override
- protected void tearDown() throws Exception {
- this.zipFile.close();
- this.zipFile = null;
- super.tearDown();
- }
-
+public class TestSnowballVocab extends LuceneTestCase {
/**
* Run all languages against their snowball vocabulary tests.
*/
@@ -82,25 +65,20 @@ public class TestSnowballVocab extends B
* For the supplied language, run the stemmer against all strings in voc.txt
* The output should be the same as the string in output.txt
*/
- private void assertCorrectOutput(String snowballLanguage, String dataDirectory)
+ private void assertCorrectOutput(final String snowballLanguage, String dataDirectory)
throws IOException {
if (VERBOSE) System.out.println("checking snowball language: " + snowballLanguage);
- TokenStream filter = new SnowballFilter(tokenizer, snowballLanguage);
- InputStream voc = zipFile.getInputStream(zipFile.getEntry(dataDirectory + "/voc.txt"));
- InputStream out = zipFile.getInputStream(zipFile.getEntry(dataDirectory + "/output.txt"));
- BufferedReader vocReader = new BufferedReader(new InputStreamReader(
- voc, "UTF-8"));
- BufferedReader outputReader = new BufferedReader(new InputStreamReader(
- out, "UTF-8"));
- String inputWord = null;
- while ((inputWord = vocReader.readLine()) != null) {
- String expectedWord = outputReader.readLine();
- assertNotNull(expectedWord);
- tokenizer.reset(new StringReader(inputWord));
- filter.reset();
- assertTokenStreamContents(filter, new String[] {expectedWord});
- }
- vocReader.close();
- outputReader.close();
+
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ Tokenizer t = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
+ }
+ };
+
+ assertVocabulary(a, getDataFile("TestSnowballVocabData.zip"),
+ dataDirectory + "/voc.txt", dataDirectory + "/output.txt");
}
}
Copied: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java Wed Jul 14 12:48:54 2010
@@ -23,10 +23,10 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
-import static org.apache.lucene.analysis.util.VocabularyAssert.*;
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Simple tests for {@link SwedishLightStemFilter}
Propchange: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/analyzers/smartcn:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/contrib/analyzers/smartcn:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/dev/trunk/modules/analysis/smartcn:942235,945090,946139,950008
/lucene/java/branches/flex_1458/contrib/analyzers/smartcn:924791,924850,930201
/lucene/java/branches/lucene_2_4/contrib/analyzers/smartcn:748824
Propchange: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/highlighter:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960349,960367,960371,960374,960611,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/contrib/highlighter:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960349,960367,960371,960374,960611,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/contrib/highlighter:924791,924850,930201
/lucene/java/branches/lucene_2_4/contrib/highlighter:748824
/lucene/java/branches/lucene_2_9/contrib/highlighter:817269-818600,825998,829134,829816,829881,831036,896850,909334,948516
Propchange: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/highlighter/src/test:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960349,960367,960371,960374,960611,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/contrib/highlighter/src/test:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960349,960367,960371,960374,960611,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/contrib/highlighter/src/test:924791,924850,930201
/lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,948516
Propchange: lucene/dev/branches/branch_3x/lucene/contrib/icu/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/icu:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/contrib/icu:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/dev/trunk/modules/analysis/icu:950008
/lucene/java/branches/flex_1458/contrib/icu:924791,924850,930201
/lucene/java/branches/lucene_2_4/contrib/icu:748824
Propchange: lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:924791,924850,930201
/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:896850,909334,948516
/lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:924483-925561
Propchange: lucene/dev/branches/branch_3x/lucene/src/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/dev/trunk/src:932749
/lucene/java/branches/flex_1458/src:924791,924850,930201
/lucene/java/branches/lucene_2_4/src:748824
Propchange: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java:924791,924850,930201
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:896850,909334,948516
/lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:924483-925561
Propchange: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:924791,924850,930201
/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:896850,909334,948516
/lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:924483-925561
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:924791,924850,930201
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,948516
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java?rev=964034&r1=964033&r2=964034&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestPorterStemFilter.java Wed Jul 14 12:48:54 2010
@@ -17,12 +17,16 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.Reader;
import java.io.StringReader;
-import java.util.zip.ZipFile;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+
+import static org.apache.lucene.analysis.VocabularyAssert.*;
/**
* Test the PorterStemFilter with Martin Porter's test data.
@@ -33,26 +37,16 @@ public class TestPorterStemFilter extend
* The output should be the same as the string in output.txt
*/
public void testPorterStemFilter() throws Exception {
- Tokenizer tokenizer = new KeywordTokenizer(new StringReader(""));
- TokenStream filter = new PorterStemFilter(tokenizer);
- ZipFile zipFile = new ZipFile(getDataFile("porterTestData.zip"));
- InputStream voc = zipFile.getInputStream(zipFile.getEntry("voc.txt"));
- InputStream out = zipFile.getInputStream(zipFile.getEntry("output.txt"));
- BufferedReader vocReader = new BufferedReader(new InputStreamReader(
- voc, "UTF-8"));
- BufferedReader outputReader = new BufferedReader(new InputStreamReader(
- out, "UTF-8"));
- String inputWord = null;
- while ((inputWord = vocReader.readLine()) != null) {
- String expectedWord = outputReader.readLine();
- assertNotNull(expectedWord);
- tokenizer.reset(new StringReader(inputWord));
- filter.reset();
- assertTokenStreamContents(filter, new String[] { expectedWord });
- }
- vocReader.close();
- outputReader.close();
- zipFile.close();
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName,
+ Reader reader) {
+ Tokenizer t = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(t, new PorterStemFilter(t));
+ }
+ };
+
+ assertVocabulary(a, getDataFile("porterTestData.zip"), "voc.txt", "output.txt");
}
public void testWithKeywordAttribute() throws IOException {
Copied: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/VocabularyAssert.java (from r964019, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/util/VocabularyAssert.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/VocabularyAssert.java?p2=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/VocabularyAssert.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/util/VocabularyAssert.java&r1=964019&r2=964034&rev=964034&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/util/VocabularyAssert.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/VocabularyAssert.java Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.util;
+package org.apache.lucene.analysis;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,7 +25,6 @@ import java.io.InputStreamReader;
import java.util.zip.ZipFile;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.junit.Assert;
/** Utility class for doing vocabulary-based stemming tests */
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java:924791,924850,930201
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,948516
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java:924791,924850,930201
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,948516
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/dev/trunk/src/test/org/apache/lucene/index:932749
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index:924791,924850,928295,930201
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index:748824
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:924791,924850,928295,930201
/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,948516
Propchange: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jul 14 12:48:54 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953407,953628,955547,955613,955615,956715,957465,957520,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,964019
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java:924791,924850,930201
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:896850,909334,948516
/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-925561