You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/01/09 14:27:36 UTC
svn commit: r1556801 [4/10] - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/
lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/
lucene/analysis/common/src/java/org/apache/lucene/analysis/br/ luc...
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestEnglishMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("EnglishMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "brick" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestKStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KStem").create(stream);
assertTokenStreamContents(stream, new String[] { "brick" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java Thu Jan 9 13:27:29 2014
@@ -34,8 +34,8 @@ import org.apache.lucene.analysis.core.K
public class TestKStemmer extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new KStemFilter(tokenizer));
}
};
@@ -57,8 +57,8 @@ public class TestKStemmer extends BaseTo
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new KStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java Thu Jan 9 13:27:29 2014
@@ -38,9 +38,8 @@ import static org.apache.lucene.analysis
public class TestPorterStemFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer t = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer( MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(t, new PorterStemFilter(t));
}
};
@@ -56,7 +55,8 @@ public class TestPorterStemFilter extend
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("yourselves");
- Tokenizer tokenizer = new MockTokenizer(new StringReader("yourselves yours"), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader("yourselves yours"));
TokenStream filter = new PorterStemFilter(new SetKeywordMarkerFilter(tokenizer, set));
assertTokenStreamContents(filter, new String[] {"yourselves", "your"});
}
@@ -69,8 +69,8 @@ public class TestPorterStemFilter extend
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PorterStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -33,7 +33,7 @@ public class TestPorterStemFilterFactory
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dogs");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PorterStem").create(stream);
assertTokenStreamContents(stream, new String[] { "dog" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -34,9 +34,8 @@ import static org.apache.lucene.analysis
public class TestSpanishLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new SpanishLightStemFilter(source));
}
};
@@ -54,8 +53,8 @@ public class TestSpanishLightStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SpanishLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@ import org.apache.lucene.analysis.util.B
public class TestSpanishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("sociedades");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("SpanishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "sociedad" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java Thu Jan 9 13:27:29 2014
@@ -26,8 +26,8 @@ import org.apache.lucene.analysis.MockTo
public class TestPersianCharFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer());
}
@Override
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java Thu Jan 9 13:27:29 2014
@@ -58,7 +58,7 @@ public class TestPersianNormalizationFil
}
private void check(final String input, final String expected) throws IOException {
- MockTokenizer tokenStream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenStream = whitespaceMockTokenizer(input);
PersianNormalizationFilter filter = new PersianNormalizationFilter(
tokenStream);
assertTokenStreamContents(filter, new String[]{expected});
@@ -67,8 +67,8 @@ public class TestPersianNormalizationFil
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PersianNormalizationFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -33,7 +33,7 @@ public class TestPersianNormalizationFil
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("ÙاÛ");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PersianNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "ÙاÙ" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.fi;
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -37,9 +36,8 @@ import static org.apache.lucene.analysis
public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FinnishLightStemFilter(source));
}
};
@@ -53,8 +51,8 @@ public class TestFinnishLightStemFilter
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("edeltäjistään"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FinnishLightStemFilter(sink));
}
@@ -70,8 +68,8 @@ public class TestFinnishLightStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FinnishLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@ import org.apache.lucene.analysis.util.B
public class TestFinnishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("aseistettujen");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("FinnishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "aseistet" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FrenchLightStemFilter(source));
}
};
@@ -183,8 +182,8 @@ public class TestFrenchLightStemFilter e
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
}
@@ -200,8 +199,8 @@ public class TestFrenchLightStemFilter e
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FrenchLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestFrenchLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("administrativement");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("FrenchLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "administratif" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Thu Jan 9 13:27:29 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(source));
}
};
@@ -62,8 +61,8 @@ public class TestFrenchMinimalStemFilter
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(sink));
}
@@ -84,8 +83,8 @@ public class TestFrenchMinimalStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FrenchMinimalStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestFrenchMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("chevaux");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("FrenchMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cheval" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java Thu Jan 9 13:27:29 2014
@@ -18,12 +18,8 @@ package org.apache.lucene.analysis.ga;
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -37,8 +33,7 @@ public class TestIrishLowerCaseFilter ex
* Test lowercase
*/
public void testIrishLowerCaseFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "nAthair tUISCE hARD"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("nAthair tUISCE hARD");
IrishLowerCaseFilter filter = new IrishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"n-athair", "t-uisce",
"hard",});
@@ -47,8 +42,8 @@ public class TestIrishLowerCaseFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IrishLowerCaseFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestIrishLowerCaseFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testCasing() throws Exception {
Reader reader = new StringReader("nAthair tUISCE hARD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IrishLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "n-athair", "t-uisce", "hard" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java Thu Jan 9 13:27:29 2014
@@ -35,8 +35,8 @@ import org.apache.lucene.analysis.util.C
public class TestGalicianMinimalStemFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer));
}
};
@@ -57,8 +57,8 @@ public class TestGalicianMinimalStemFilt
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("elefantes"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GalicianMinimalStemFilter(sink));
}
@@ -74,8 +74,8 @@ public class TestGalicianMinimalStemFilt
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestGalicianMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("elefantes");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GalicianMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "elefante" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java Thu Jan 9 13:27:29 2014
@@ -36,9 +36,8 @@ import org.apache.lucene.analysis.standa
public class TestGalicianStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new GalicianStemFilter(result));
}
@@ -53,8 +52,8 @@ public class TestGalicianStemFilter exte
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GalicianStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestGalicianStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("cariñosa");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GalicianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cariñ" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java Thu Jan 9 13:27:29 2014
@@ -21,7 +21,9 @@ import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.analysis.util.TokenizerFactory;
/**
* Simple tests to ensure the Hindi filter Factories are working.
@@ -32,7 +34,8 @@ public class TestHindiFilters extends Ba
*/
public void testIndicNormalizer() throws Exception {
Reader reader = new StringReader("তà§â à¤
ाà¥à¤°");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "à§", "à¤à¤°" });
}
@@ -42,7 +45,8 @@ public class TestHindiFilters extends Ba
*/
public void testHindiNormalizer() throws Exception {
Reader reader = new StringReader("à¥à¤¿à¤¤à¤¾à¤¬");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
assertTokenStreamContents(stream, new String[] {"à¤à¤¿à¤¤à¤¾à¤¬"});
@@ -53,7 +57,8 @@ public class TestHindiFilters extends Ba
*/
public void testStemmer() throws Exception {
Reader reader = new StringReader("à¤à¤¿à¤¤à¤¾à¤¬à¥à¤");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
stream = tokenFilterFactory("HindiStem").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java Thu Jan 9 13:27:29 2014
@@ -62,7 +62,7 @@ public class TestHindiNormalizer extends
check("à¤à¤à¤à¥ ॡà¤à¤à¥à¥à¥à¥£à¥à¥", "à¤
à¤à¤à¤à¤à¤à¤à¤¿à¥à¥à¥¢à¥à¥");
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new HindiNormalizationFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -70,8 +70,8 @@ public class TestHindiNormalizer extends
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HindiNormalizationFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java Thu Jan 9 13:27:29 2014
@@ -18,12 +18,9 @@ package org.apache.lucene.analysis.hi;
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -84,7 +81,7 @@ public class TestHindiStemmer extends Ba
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new HindiStemFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -92,8 +89,8 @@ public class TestHindiStemmer extends Ba
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HindiStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new HungarianLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@ public class TestHungarianLightStemFilte
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("babakocsi"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new HungarianLightStemFilter(sink));
}
@@ -65,8 +64,8 @@ public class TestHungarianLightStemFilte
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HungarianLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestHungarianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("házakat");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("HungarianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "haz" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java Thu Jan 9 13:27:29 2014
@@ -18,8 +18,6 @@ package org.apache.lucene.analysis.hunsp
import java.io.IOException;
import java.io.InputStream;
-import java.io.Reader;
-import java.io.StringReader;
import java.text.ParseException;
import java.util.Arrays;
@@ -56,13 +54,13 @@ public class HunspellStemFilterTest ext
* Simple test for KeywordAttribute
*/
public void testKeywordAttribute() throws IOException {
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
+ MockTokenizer tokenizer = whitespaceMockTokenizer("lucene is awesome");
tokenizer.setEnableChecks(true);
HunspellStemFilter filter = new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "lucen", "is", "awesome"}, new int[] {1, 0, 1, 1});
// assert with keywork marker
- tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
+ tokenizer = whitespaceMockTokenizer("lucene is awesome");
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList("Lucene"), true);
filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
@@ -73,8 +71,8 @@ public class HunspellStemFilterTest ext
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
@@ -84,8 +82,8 @@ public class HunspellStemFilterTest ext
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestHunspellStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("abc");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("HunspellStem",
"dictionary", "test.dic",
"affix", "test.aff").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@ public class TestIndonesianStemFilterFac
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dibukukannya");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndonesianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "buku" });
}
@@ -43,7 +45,8 @@ public class TestIndonesianStemFilterFac
*/
public void testStemmingInflectional() throws Exception {
Reader reader = new StringReader("dibukukannya");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndonesianStem", "stemDerivational", "false").create(stream);
assertTokenStreamContents(stream, new String[] { "dibukukan" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java Thu Jan 9 13:27:29 2014
@@ -32,8 +32,8 @@ public class TestIndonesianStemmer exten
/* full stemming, no stopwords */
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
}
};
@@ -113,8 +113,8 @@ public class TestIndonesianStemmer exten
/* inflectional-only stemming */
Analyzer b = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer, false));
}
};
@@ -136,8 +136,8 @@ public class TestIndonesianStemmer exten
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java Thu Jan 9 13:27:29 2014
@@ -47,7 +47,8 @@ public class TestIndicNormalizer extends
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);;
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);;
+ tokenizer.setReader(new StringReader(input));
TokenFilter tf = new IndicNormalizationFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -55,8 +56,8 @@ public class TestIndicNormalizer extends
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndicNormalizationFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -34,9 +34,8 @@ import static org.apache.lucene.analysis
public class TestItalianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new ItalianLightStemFilter(source));
}
};
@@ -54,8 +53,8 @@ public class TestItalianLightStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ItalianLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestItalianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("ragazzo ragazzi");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ItalianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "ragazz", "ragazz" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@ import org.apache.lucene.analysis.util.B
public class TestLatvianStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("tirgiem tirgus");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("LatvianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "tirg", "tirg" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java Thu Jan 9 13:27:29 2014
@@ -32,8 +32,8 @@ import org.apache.lucene.analysis.core.K
public class TestLatvianStemmer extends BaseTokenStreamTestCase {
private Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer));
}
};
@@ -273,8 +273,8 @@ public class TestLatvianStemmer extends
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java Thu Jan 9 13:27:29 2014
@@ -26,8 +26,6 @@ import org.apache.lucene.analysis.core.K
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
@@ -36,10 +34,9 @@ public class TestASCIIFoldingFilter exte
// testLain1Accents() is a copy of TestLatin1AccentFilter.testU().
public void testLatin1Accents() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader
- ("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à Ã"
+ TokenStream stream = whitespaceMockTokenizer("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à Ã"
+" à à à à à à Šà à à à à à Ÿ à á â ã ä å æ ç è é ê ë ì à î ï ij"
- +" ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"), MockTokenizer.WHITESPACE, false);
+ +" ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬");
ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
@@ -1897,7 +1894,7 @@ public class TestASCIIFoldingFilter exte
expectedOutputTokens.add(expected.toString());
}
- TokenStream stream = new MockTokenizer(new StringReader(inputText.toString()), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(inputText.toString());
ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
Iterator<String> expectedIter = expectedOutputTokens.iterator();
@@ -1918,8 +1915,8 @@ public class TestASCIIFoldingFilter exte
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
}
};
@@ -1929,8 +1926,8 @@ public class TestASCIIFoldingFilter exte
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java Thu Jan 9 13:27:29 2014
@@ -107,8 +107,10 @@ public class TestCapitalizationFilter ex
boolean onlyFirstWord, CharArraySet keep, boolean forceFirstLetter,
Collection<char[]> okPrefix, int minWordLength, int maxWordCount,
int maxTokenLength) throws IOException {
- assertCapitalizesTo(new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
- expected, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength,
+ final MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader(input));
+ assertCapitalizesTo(tokenizer,
+ expected, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength,
maxWordCount, maxTokenLength);
}
@@ -116,7 +118,9 @@ public class TestCapitalizationFilter ex
boolean onlyFirstWord, CharArraySet keep, boolean forceFirstLetter,
Collection<char[]> okPrefix, int minWordLength, int maxWordCount,
int maxTokenLength) throws IOException {
- assertCapitalizesTo(new MockTokenizer(new StringReader(input), MockTokenizer.KEYWORD, false),
+ final MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ tokenizer.setReader(new StringReader(input));
+ assertCapitalizesTo(tokenizer,
new String[] { expected }, onlyFirstWord, keep, forceFirstLetter, okPrefix,
minWordLength, maxWordCount, maxTokenLength);
}
@@ -126,8 +130,8 @@ public class TestCapitalizationFilter ex
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new CapitalizationFilter(tokenizer));
}
};
@@ -138,8 +142,8 @@ public class TestCapitalizationFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CapitalizationFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -28,7 +28,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true").create(stream);
@@ -37,7 +37,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization2() throws Exception {
Reader reader = new StringReader("and");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -48,7 +48,7 @@ public class TestCapitalizationFilterFac
/** first is forced, but it's not a keep word, either */
public void testCapitalization3() throws Exception {
Reader reader = new StringReader("AnD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -58,7 +58,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization4() throws Exception {
Reader reader = new StringReader("AnD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -68,7 +68,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization5() throws Exception {
Reader reader = new StringReader("big");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -78,7 +78,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization6() throws Exception {
Reader reader = new StringReader("BIG");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -88,7 +88,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization7() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -98,7 +98,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization8() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -108,7 +108,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization9() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -119,7 +119,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization10() throws Exception {
Reader reader = new StringReader("McKinley");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -131,7 +131,7 @@ public class TestCapitalizationFilterFac
/** using "McK" as okPrefix */
public void testCapitalization11() throws Exception {
Reader reader = new StringReader("McKinley");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -144,7 +144,7 @@ public class TestCapitalizationFilterFac
/** test with numbers */
public void testCapitalization12() throws Exception {
Reader reader = new StringReader("1st 2nd third");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -156,7 +156,7 @@ public class TestCapitalizationFilterFac
public void testCapitalization13() throws Exception {
Reader reader = new StringReader("the The the");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -168,7 +168,7 @@ public class TestCapitalizationFilterFac
public void testKeepIgnoreCase() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"keepIgnoreCase", "true",
@@ -180,7 +180,7 @@ public class TestCapitalizationFilterFac
public void testKeepIgnoreCase2() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"keepIgnoreCase", "true",
@@ -192,7 +192,7 @@ public class TestCapitalizationFilterFac
public void testKeepIgnoreCase3() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keepIgnoreCase", "true",
"onlyFirstWord", "true",
@@ -208,7 +208,7 @@ public class TestCapitalizationFilterFac
*/
public void testMinWordLength() throws Exception {
Reader reader = new StringReader("helo testing");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"onlyFirstWord", "true",
"minWordLength", "5").create(stream);
@@ -221,7 +221,7 @@ public class TestCapitalizationFilterFac
*/
public void testMaxWordCount() throws Exception {
Reader reader = new StringReader("one two three four");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxWordCount", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "One", "Two", "Three", "Four" });
@@ -232,7 +232,7 @@ public class TestCapitalizationFilterFac
*/
public void testMaxWordCount2() throws Exception {
Reader reader = new StringReader("one two three four");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxWordCount", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "one two three four" });
@@ -245,7 +245,7 @@ public class TestCapitalizationFilterFac
*/
public void testMaxTokenLength() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxTokenLength", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "this", "is", "A", "test" });
@@ -256,7 +256,7 @@ public class TestCapitalizationFilterFac
*/
public void testForceFirstLetterWithKeep() throws Exception {
Reader reader = new StringReader("kitten");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"forceFirstLetter", "true").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java Thu Jan 9 13:27:29 2014
@@ -31,8 +31,7 @@ import org.apache.lucene.util._TestUtil;
public class TestCodepointCountFilter extends BaseTokenStreamTestCase {
public void testFilterWithPosIncr() throws Exception {
- TokenStream stream = new MockTokenizer(
- new StringReader("short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("short toolong evenmuchlongertext a ab toolong foo");
CodepointCountFilter filter = new CodepointCountFilter(TEST_VERSION_CURRENT, stream, 2, 6);
assertTokenStreamContents(filter,
new String[]{"short", "ab", "foo"},
@@ -43,8 +42,8 @@ public class TestCodepointCountFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CodepointCountFilter(TEST_VERSION_CURRENT, tokenizer, 0, 5));
}
};
@@ -58,7 +57,8 @@ public class TestCodepointCountFilter ex
int max = _TestUtil.nextInt(random(), 0, 100);
int count = text.codePointCount(0, text.length());
boolean expected = count >= min && count <= max;
- TokenStream stream = new KeywordTokenizer(new StringReader(text));
+ TokenStream stream = new KeywordTokenizer();
+ ((Tokenizer)stream).setReader(new StringReader(text));
stream = new CodepointCountFilter(TEST_VERSION_CURRENT, stream, min, max);
stream.reset();
assertEquals(expected, stream.incrementToken());
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,13 +22,15 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
public class TestCodepointCountFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testPositionIncrements() throws Exception {
Reader reader = new StringReader("foo foobar super-duper-trooper");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("CodepointCount",
"min", "4",
"max", "10").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java Thu Jan 9 13:27:29 2014
@@ -35,7 +35,8 @@ public class TestHyphenatedWordsFilter e
public void testHyphenatedWords() throws Exception {
String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecologi-\ncal";
// first test
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
ts = new HyphenatedWordsFilter(ts);
assertTokenStreamContents(ts,
new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecological" });
@@ -45,17 +46,19 @@ public class TestHyphenatedWordsFilter e
* Test that HyphenatedWordsFilter behaves correctly with a final hyphen
*/
public void testHyphenAtEnd() throws Exception {
- String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
+ String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
// first test
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
- ts = new HyphenatedWordsFilter(ts);
- assertTokenStreamContents(ts,
- new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
+ ts = new HyphenatedWordsFilter(ts);
+ assertTokenStreamContents(ts,
+ new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
}
public void testOffsets() throws Exception {
String input = "abc- def geh 1234- 5678-";
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
ts = new HyphenatedWordsFilter(ts);
assertTokenStreamContents(ts,
new String[] { "abcdef", "geh", "12345678-" },
@@ -68,8 +71,8 @@ public class TestHyphenatedWordsFilter e
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new HyphenatedWordsFilter(tokenizer));
}
};
@@ -80,8 +83,8 @@ public class TestHyphenatedWordsFilter e
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HyphenatedWordsFilter(tokenizer));
}
};