You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/19 17:59:32 UTC
svn commit: r1363400 [4/31] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/copyright/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/maven/ dev-tools/maven/lucene/...
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java Thu Jul 19 15:58:54 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -53,6 +56,19 @@ public class TestGermanMinimalStemFilter
checkOneTerm(analyzer, "äpfel", "apfel");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new GermanMinimalStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "sängerinnen", "sängerinnen");
+ }
+
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("deminimaltestdata.zip"), "deminimal.txt");
@@ -60,7 +76,7 @@ public class TestGermanMinimalStemFilter
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java Thu Jul 19 15:58:54 2012
@@ -64,7 +64,7 @@ public class TestGermanNormalizationFilt
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Thu Jul 19 15:58:54 2012
@@ -23,9 +23,13 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -58,9 +62,22 @@ public class TestGermanStemFilter extend
assertAnalyzesTo(analyzer, "", new String[] { "" });
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new GermanStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "sängerinnen", "sängerinnen");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java Thu Jul 19 15:58:54 2012
@@ -66,6 +66,6 @@ public class GreekAnalyzerTest extends B
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new GreekAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new GreekAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -55,6 +55,6 @@ public class TestEnglishAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new EnglishAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new EnglishAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java Thu Jul 19 15:58:54 2012
@@ -54,7 +54,7 @@ public class TestEnglishMinimalStemFilte
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java Thu Jul 19 15:58:54 2012
@@ -42,7 +42,7 @@ public class TestKStemmer extends BaseTo
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
/**
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java Thu Jul 19 15:58:54 2012
@@ -63,7 +63,7 @@ public class TestPorterStemFilter extend
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestSpanishAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new SpanishAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new SpanishAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java Thu Jul 19 15:58:54 2012
@@ -48,7 +48,7 @@ public class TestSpanishLightStemFilter
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/eu/TestBasqueAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestBasqueAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new BasqueAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new BasqueAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -224,6 +224,6 @@ public class TestPersianAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new PersianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new PersianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestFinnishAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new FinnishAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new FinnishAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Thu Jul 19 15:58:54 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -46,9 +49,22 @@ public class TestFinnishLightStemFilter
assertVocabulary(analyzer, getDataFile("filighttestdata.zip"), "filight.txt");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("edeltäjistään"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new FinnishLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "edeltäjistään", "edeltäjistään");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -164,7 +164,7 @@ public class TestFrenchAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new FrenchAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new FrenchAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
/** test accent-insensitive */
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Thu Jul 19 15:58:54 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -176,9 +179,22 @@ public class TestFrenchLightStemFilter e
assertVocabulary(analyzer, getDataFile("frlighttestdata.zip"), "frlight.txt");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "chevaux", "chevaux");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Thu Jul 19 15:58:54 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,6 +58,19 @@ public class TestFrenchMinimalStemFilter
checkOneTerm(analyzer, "baron", "baron");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new FrenchMinimalStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "chevaux", "chevaux");
+ }
+
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("frminimaltestdata.zip"), "frminimal.txt");
@@ -62,7 +78,7 @@ public class TestFrenchMinimalStemFilter
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -66,6 +66,6 @@ public class TestIrishAnalyzer extends B
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new IrishAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new IrishAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestGalicianAnalyzer extend
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new GalicianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new GalicianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java Thu Jul 19 15:58:54 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
/**
* Simple tests for {@link GalicianMinimalStemmer}
@@ -50,9 +53,22 @@ public class TestGalicianMinimalStemFilt
checkOneTerm(a, "barcelonês", "barcelonês");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("elefantes"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new GalicianMinimalStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "elefantes", "elefantes");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -47,6 +47,6 @@ public class TestHindiAnalyzer extends B
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new HindiAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new HindiAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestHungarianAnalyzer exten
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new HungarianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new HungarianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java Thu Jul 19 15:58:54 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -46,6 +49,19 @@ public class TestHungarianLightStemFilte
assertVocabulary(analyzer, getDataFile("hulighttestdata.zip"), "hulight.txt");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("babakocsi"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new HungarianLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "babakocsi", "babakocsi");
+ }
+
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java Thu Jul 19 15:58:54 2012
@@ -17,19 +17,98 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.Version;
-import org.junit.Assert;
-import org.junit.Test;
-
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.util.Arrays;
-import static junit.framework.Assert.assertEquals;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Assert;
+import org.junit.Test;
public class HunspellDictionaryTest extends LuceneTestCase {
+
+ private class CloseCheckInputStream extends InputStream {
+ private InputStream delegate;
+
+ private boolean closed = false;
+
+ public CloseCheckInputStream(InputStream delegate) {
+ super();
+ this.delegate = delegate;
+ }
+
+ public int read() throws IOException {
+ return delegate.read();
+ }
+
+ public int hashCode() {
+ return delegate.hashCode();
+ }
+
+ public int read(byte[] b) throws IOException {
+ return delegate.read(b);
+ }
+
+ public boolean equals(Object obj) {
+ return delegate.equals(obj);
+ }
+
+ public int read(byte[] b, int off, int len) throws IOException {
+ return delegate.read(b, off, len);
+ }
+
+ public long skip(long n) throws IOException {
+ return delegate.skip(n);
+ }
+
+ public String toString() {
+ return delegate.toString();
+ }
+
+ public int available() throws IOException {
+ return delegate.available();
+ }
+
+ public void close() throws IOException {
+ this.closed = true;
+ delegate.close();
+ }
+
+ public void mark(int readlimit) {
+ delegate.mark(readlimit);
+ }
+
+ public void reset() throws IOException {
+ delegate.reset();
+ }
+
+ public boolean markSupported() {
+ return delegate.markSupported();
+ }
+
+ public boolean isClosed() {
+ return this.closed;
+ }
+
+ }
+
+ @Test
+ public void testResourceCleanup() throws IOException, ParseException {
+ CloseCheckInputStream affixStream = new CloseCheckInputStream(getClass().getResourceAsStream("testCompressed.aff"));
+ CloseCheckInputStream dictStream = new CloseCheckInputStream(getClass().getResourceAsStream("testCompressed.dic"));
+
+ new HunspellDictionary(affixStream, dictStream, TEST_VERSION_CURRENT);
+
+ assertFalse(affixStream.isClosed());
+ assertFalse(dictStream.isClosed());
+
+ affixStream.close();
+ dictStream.close();
+
+ assertTrue(affixStream.isClosed());
+ assertTrue(dictStream.isClosed());
+ }
@Test
public void testHunspellDictionary_loadDicAff() throws IOException, ParseException {
@@ -40,7 +119,7 @@ public class HunspellDictionaryTest exte
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).size());
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
-
+
affixStream.close();
dictStream.close();
}
@@ -54,7 +133,7 @@ public class HunspellDictionaryTest exte
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).size());
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
-
+
affixStream.close();
dictStream.close();
}
@@ -69,7 +148,9 @@ public class HunspellDictionaryTest exte
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
//strict parsing disabled: malformed rule is not loaded
- assertNull(dictionary.lookupPrefix(new char[]{'a'}, 0, 1));
+ assertNull(dictionary.lookupPrefix(new char[]{'a'}, 0, 1));
+ affixStream.close();
+ dictStream.close();
affixStream = getClass().getResourceAsStream("testWrongAffixRule.aff");
dictStream = getClass().getResourceAsStream("test.dic");
@@ -81,7 +162,7 @@ public class HunspellDictionaryTest exte
Assert.assertEquals("The affix file contains a rule with less than five elements", e.getMessage());
Assert.assertEquals(23, e.getErrorOffset());
}
-
+
affixStream.close();
dictStream.close();
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java Thu Jul 19 15:58:54 2012
@@ -72,7 +72,7 @@ public class HunspellStemFilterTest ext
return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY));
}
};
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/hy/TestArmenianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestArmenianAnalyzer extend
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new ArmenianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new ArmenianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestIndonesianAnalyzer exte
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new IndonesianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new IndonesianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -54,7 +54,7 @@ public class TestItalianAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new ItalianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new ItalianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
/** test that the elisionfilter is working */
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java Thu Jul 19 15:58:54 2012
@@ -48,7 +48,7 @@ public class TestItalianLightStemFilter
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestLatvianAnalyzer extends
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new LatvianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new LatvianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java Thu Jul 19 15:58:54 2012
@@ -1923,7 +1923,7 @@ public class TestASCIIFoldingFilter exte
return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java Thu Jul 19 15:58:54 2012
@@ -132,7 +132,7 @@ public class TestCapitalizationFilter ex
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java Thu Jul 19 15:58:54 2012
@@ -74,7 +74,7 @@ public class TestHyphenatedWordsFilter e
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java Thu Jul 19 15:58:54 2012
@@ -77,6 +77,6 @@ public class TestKeepWordFilter extends
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java Thu Jul 19 15:58:54 2012
@@ -79,7 +79,7 @@ public class TestKeywordMarkerFilter ext
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (!keywordAttr.isKeyword()) {
- final String term = termAtt.toString().toLowerCase(Locale.ENGLISH);
+ final String term = termAtt.toString().toLowerCase(Locale.ROOT);
termAtt.setEmpty().append(term);
}
return true;
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java Thu Jul 19 15:58:54 2012
@@ -123,7 +123,7 @@ public class TestTrimFilter extends Base
return new TokenStreamComponents(tokenizer, new TrimFilter(tokenizer, false));
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
Analyzer b = new Analyzer() {
@@ -133,7 +133,7 @@ public class TestTrimFilter extends Base
return new TokenStreamComponents(tokenizer, new TrimFilter(tokenizer, true));
}
};
- checkRandomData(random(), b, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Thu Jul 19 15:58:54 2012
@@ -116,7 +116,7 @@ public class EdgeNGramTokenFilterTest ex
WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
- tokenizer.reset(new StringReader("abcde"));
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
}
@@ -147,20 +147,20 @@ public class EdgeNGramTokenFilterTest ex
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
- new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 2, 15));
+ new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 2, 4));
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
Analyzer b = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
- new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.BACK, 2, 15));
+ new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.BACK, 2, 4));
}
};
- checkRandomData(random(), b, 10000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER, 20, false, false);
}
public void testEmptyTerm() throws Exception {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Thu Jul 19 15:58:54 2012
@@ -96,7 +96,7 @@ public class EdgeNGramTokenizerTest exte
public void testReset() throws Exception {
EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 1, 3);
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
- tokenizer.reset(new StringReader("abcde"));
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
}
@@ -105,21 +105,21 @@ public class EdgeNGramTokenizerTest exte
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.FRONT, 2, 15);
+ Tokenizer tokenizer = new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.FRONT, 2, 4);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER, 20, false, false);
- checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 8192, false, false);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random(), a, 100*RANDOM_MULTIPLIER, 8192, false, false);
Analyzer b = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.BACK, 2, 15);
+ Tokenizer tokenizer = new EdgeNGramTokenizer(reader, EdgeNGramTokenizer.Side.BACK, 2, 4);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), b, 10000*RANDOM_MULTIPLIER, 20, false, false);
- checkRandomData(random(), b, 200*RANDOM_MULTIPLIER, 8192, false, false);
+ checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random(), b, 100*RANDOM_MULTIPLIER, 8192, false, false);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Thu Jul 19 15:58:54 2012
@@ -98,7 +98,7 @@ public class NGramTokenFilterTest extend
WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
NGramTokenFilter filter = new NGramTokenFilter(tokenizer, 1, 1);
assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
- tokenizer.reset(new StringReader("abcde"));
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
}
@@ -129,10 +129,10 @@ public class NGramTokenFilterTest extend
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
- new NGramTokenFilter(tokenizer, 2, 15));
+ new NGramTokenFilter(tokenizer, 2, 4));
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
}
public void testEmptyTerm() throws Exception {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Thu Jul 19 15:58:54 2012
@@ -24,6 +24,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.LuceneTestCase.Slow;
/**
* Tests {@link NGramTokenizer} for correctness.
@@ -89,7 +90,7 @@ public class NGramTokenizerTest extends
public void testReset() throws Exception {
NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 1);
assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
- tokenizer.reset(new StringReader("abcde"));
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
}
@@ -98,11 +99,11 @@ public class NGramTokenizerTest extends
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new NGramTokenizer(reader, 2, 15);
+ Tokenizer tokenizer = new NGramTokenizer(reader, 2, 4);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER, 20, false, false);
- checkRandomData(random(), a, 200*RANDOM_MULTIPLIER, 8192, false, false);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random(), a, 50*RANDOM_MULTIPLIER, 1027, false, false);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java Thu Jul 19 15:58:54 2012
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
@@ -147,6 +148,12 @@ public class TestDutchStemmer extends Ba
checkOneTerm(a, "fiets", "fiets");
}
+ public void testEmptyStemDictionary() throws IOException {
+ DutchAnalyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET,
+ CharArraySet.EMPTY_SET, CharArrayMap.<String>emptyMap());
+ checkOneTerm(a, "fiets", "fiet");
+ }
+
/**
* Test that stopwords are not case sensitive
*/
@@ -161,7 +168,7 @@ public class TestDutchStemmer extends Ba
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new DutchAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new DutchAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
\ No newline at end of file
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestNorwegianAnalyzer exten
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new NorwegianAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new NorwegianAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java Thu Jul 19 15:58:54 2012
@@ -25,8 +25,11 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -47,11 +50,24 @@ public class TestNorwegianLightStemFilte
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_light.txt")));
}
+
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "sekretæren", "sekretæren");
+ }
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Random random = random();
- checkRandomData(random, analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random, analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java Thu Jul 19 15:58:54 2012
@@ -25,8 +25,11 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -47,11 +50,24 @@ public class TestNorwegianMinimalStemFil
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_minimal.txt")));
}
+
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "sekretæren", "sekretæren");
+ }
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Random random = random();
- checkRandomData(random, analyzer, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random, analyzer, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java Thu Jul 19 15:58:54 2012
@@ -23,7 +23,6 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
@@ -123,7 +122,7 @@ public class TestPathHierarchyTokenizer
builder.add("\\", "/");
NormalizeCharMap normMap = builder.build();
String path = "c:\\a\\b\\c";
- CharStream cs = new MappingCharFilter(normMap, new StringReader(path));
+ Reader cs = new MappingCharFilter(normMap, new StringReader(path));
PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs );
assertTokenStreamContents(t,
new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
@@ -208,7 +207,7 @@ public class TestPathHierarchyTokenizer
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
/** blast some random large strings through the analyzer */
@@ -221,6 +220,6 @@ public class TestPathHierarchyTokenizer
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 1027);
+ checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java Thu Jul 19 15:58:54 2012
@@ -169,7 +169,7 @@ public class TestReversePathHierarchyTok
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
}
/** blast some random large strings through the analyzer */
@@ -182,6 +182,6 @@ public class TestReversePathHierarchyTok
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 1027);
+ checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java Thu Jul 19 15:58:54 2012
@@ -26,8 +26,7 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -102,8 +101,8 @@ public class TestPatternReplaceCharFilte
private void checkOutput(String input, String pattern, String replacement,
String expectedOutput, String expectedIndexMatchedOutput) throws IOException {
- CharStream cs = new PatternReplaceCharFilter(pattern(pattern), replacement,
- CharReader.get(new StringReader(input)));
+ CharFilter cs = new PatternReplaceCharFilter(pattern(pattern), replacement,
+ new StringReader(input));
StringBuilder output = new StringBuilder();
for (int chr = cs.read(); chr > 0; chr = cs.read()) {
@@ -138,8 +137,8 @@ public class TestPatternReplaceCharFilte
// this is test.
public void testNothingChange() throws IOException {
final String BLOCK = "this is test.";
- CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1$2$3",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1$2$3",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "this", "is", "test." },
@@ -152,8 +151,8 @@ public class TestPatternReplaceCharFilte
// aa bb cc
public void testReplaceByEmpty() throws IOException {
final String BLOCK = "aa bb cc";
- CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[] {});
}
@@ -163,8 +162,8 @@ public class TestPatternReplaceCharFilte
// aa#bb#cc
public void test1block1matchSameLength() throws IOException {
final String BLOCK = "aa bb cc";
- CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2#$3",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2#$3",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "aa#bb#cc" },
@@ -179,8 +178,8 @@ public class TestPatternReplaceCharFilte
// aa##bb###cc dd
public void test1block1matchLonger() throws IOException {
final String BLOCK = "aa bb cc dd";
- CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1##$2###$3",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1##$2###$3",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "aa##bb###cc", "dd" },
@@ -194,8 +193,8 @@ public class TestPatternReplaceCharFilte
// aa aa
public void test1block2matchLonger() throws IOException {
final String BLOCK = " a a";
- CharStream cs = new PatternReplaceCharFilter( pattern("a"), "aa",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("a"), "aa",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "aa", "aa" },
@@ -210,8 +209,8 @@ public class TestPatternReplaceCharFilte
// aa#bb dd
public void test1block1matchShorter() throws IOException {
final String BLOCK = "aa bb cc dd";
- CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "aa#bb", "dd" },
@@ -226,8 +225,8 @@ public class TestPatternReplaceCharFilte
// aa bb cc --- aa bb aa bb cc
public void test1blockMultiMatches() throws IOException {
final String BLOCK = " aa bb cc --- aa bb aa bb cc";
- CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1 $2 $3",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1 $2 $3",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "aa", "bb", "cc", "---", "aa", "bb", "aa", "bb", "cc" },
@@ -246,8 +245,8 @@ public class TestPatternReplaceCharFilte
public void test2blocksMultiMatches() throws IOException {
final String BLOCK = " aa bb cc --- aa bb aa. bb aa bb cc";
- CharStream cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)"), "$1##$2",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)"), "$1##$2",
+ new StringReader( BLOCK ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[] { "aa##bb", "cc", "---", "aa##bb", "aa.", "bb", "aa##bb", "cc" },
@@ -262,8 +261,8 @@ public class TestPatternReplaceCharFilte
// aa b - c . --- b aa . c c b
public void testChain() throws IOException {
final String BLOCK = " a bb - ccc . --- bb a . ccc ccc bb";
- CharStream cs = new PatternReplaceCharFilter( pattern("a"), "aa",
- CharReader.get( new StringReader( BLOCK ) ) );
+ CharFilter cs = new PatternReplaceCharFilter( pattern("a"), "aa",
+ new StringReader( BLOCK ) );
cs = new PatternReplaceCharFilter( pattern("bb"), "b", cs );
cs = new PatternReplaceCharFilter( pattern("ccc"), "c", cs );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
@@ -315,7 +314,7 @@ public class TestPatternReplaceCharFilte
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new PatternReplaceCharFilter(p, replacement, CharReader.get(reader));
+ return new PatternReplaceCharFilter(p, replacement, reader);
}
};
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java Thu Jul 19 15:58:54 2012
@@ -93,7 +93,7 @@ public class TestPatternReplaceFilter ex
return new TokenStreamComponents(tokenizer, filter);
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
Analyzer b = new Analyzer() {
@Override
@@ -103,7 +103,7 @@ public class TestPatternReplaceFilter ex
return new TokenStreamComponents(tokenizer, filter);
}
};
- checkRandomData(random(), b, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER);
}
public void testEmptyTerm() throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java Thu Jul 19 15:58:54 2012
@@ -26,8 +26,7 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
@@ -83,7 +82,7 @@ public class TestPatternTokenizer extend
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
builder.add("ü", "ü");
NormalizeCharMap normMap = builder.build();
- CharStream charStream = new MappingCharFilter( normMap, CharReader.get( new StringReader( INPUT ) ) );
+ CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
// create PatternTokenizer
TokenStream stream = new PatternTokenizer(charStream, Pattern.compile("[,;/\\s]+"), -1);
@@ -93,7 +92,7 @@ public class TestPatternTokenizer extend
new int[] { 12, 25, 28, 33 },
INPUT.length());
- charStream = new MappingCharFilter( normMap, CharReader.get( new StringReader( INPUT ) ) );
+ charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
stream = new PatternTokenizer(charStream, Pattern.compile("Günther"), 0);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther" },
@@ -138,7 +137,7 @@ public class TestPatternTokenizer extend
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
Analyzer b = new Analyzer() {
@Override
@@ -152,6 +151,6 @@ public class TestPatternTokenizer extend
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), b, 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER);
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java Thu Jul 19 15:58:54 2012
@@ -17,8 +17,6 @@ package org.apache.lucene.analysis.posit
* limitations under the License.
*/
-import java.io.IOException;
-
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.shingle.ShingleFilter;
@@ -38,7 +36,7 @@ public class PositionFilterTest extends
}
@Override
- public final boolean incrementToken() throws IOException {
+ public final boolean incrementToken() {
clearAttributes();
if (index < testToken.length) {
termAtt.setEmpty().append(testToken[index++]);
Modified: lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseAnalyzer.java Thu Jul 19 15:58:54 2012
@@ -51,6 +51,6 @@ public class TestPortugueseAnalyzer exte
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new PortugueseAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ checkRandomData(random(), new PortugueseAnalyzer(TEST_VERSION_CURRENT), 1000*RANDOM_MULTIPLIER);
}
}