You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/01/09 14:27:36 UTC
svn commit: r1556801 [7/10] - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/
lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/
lucene/analysis/common/src/java/org/apache/lucene/analysis/br/ luc...
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Thu Jan 9 13:27:29 2014
@@ -18,16 +18,14 @@ package org.apache.lucene.analysis.snowb
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
+
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@@ -35,15 +33,15 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Version;
+
public class TestSnowball extends BaseTokenStreamTestCase {
public void testEnglish() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer();
return new TokenStreamComponents(tokenizer, new SnowballFilter(tokenizer, "English"));
}
};
@@ -109,8 +107,8 @@ public class TestSnowball extends BaseTo
for (final String lang : SNOWBALL_LANGS) {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SnowballFilter(tokenizer, lang));
}
};
@@ -127,8 +125,8 @@ public class TestSnowball extends BaseTo
public void checkRandomStrings(final String snowballLanguage) throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer();
return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballPorterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballPorterFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballPorterFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballPorterFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -39,7 +39,7 @@ public class TestSnowballPorterFilterFac
}
Reader reader = new StringReader(text);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("SnowballPorter", "language", "English").create(stream);
assertTokenStreamContents(stream, gold);
}
@@ -49,7 +49,7 @@ public class TestSnowballPorterFilterFac
*/
public void testProtected() throws Exception {
Reader reader = new StringReader("ridding of some stemming");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("SnowballPorter", TEST_VERSION_CURRENT,
new StringMockResourceLoader("ridding"),
"protected", "protwords.txt",
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java Thu Jan 9 13:27:29 2014
@@ -70,9 +70,8 @@ public class TestSnowballVocab extends L
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer t = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new KeywordTokenizer();
return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java Thu Jan 9 13:27:29 2014
@@ -34,9 +34,10 @@ public class TestStandardFactories exten
*/
public void testStandardTokenizer() throws Exception {
Reader reader = new StringReader("Wha\u0301t's this thing do?");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "Wha\u0301t's", "this", "thing", "do" });
+ Tokenizer stream = tokenizerFactory("Standard").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"Wha\u0301t's", "this", "thing", "do"});
}
public void testStandardTokenizerMaxTokenLength() throws Exception {
@@ -48,9 +49,10 @@ public class TestStandardFactories exten
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
Tokenizer stream = tokenizerFactory("Standard",
- "maxTokenLength", "1000").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "one", "two", "three", longWord, "four", "five", "six" });
+ "maxTokenLength", "1000").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"one", "two", "three", longWord, "four", "five", "six"});
}
/**
@@ -58,9 +60,10 @@ public class TestStandardFactories exten
*/
public void testClassicTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Classic").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "What's", "this", "thing", "do" });
+ Tokenizer stream = tokenizerFactory("Classic").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"What's", "this", "thing", "do"});
}
public void testClassicTokenizerMaxTokenLength() throws Exception {
@@ -72,9 +75,10 @@ public class TestStandardFactories exten
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
Tokenizer stream = tokenizerFactory("Classic",
- "maxTokenLength", "1000").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "one", "two", "three", longWord, "four", "five", "six" });
+ "maxTokenLength", "1000").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"one", "two", "three", longWord, "four", "five", "six"});
}
/**
@@ -82,8 +86,9 @@ public class TestStandardFactories exten
*/
public void testStandardFilter() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Classic").create(reader);
- stream = tokenFilterFactory("Classic").create(stream);
+ Tokenizer tokenizer = tokenizerFactory("Classic").create();
+ tokenizer.setReader(reader);
+ TokenStream stream = tokenFilterFactory("Classic").create(tokenizer);
assertTokenStreamContents(stream,
new String[] { "What", "this", "thing", "do" });
}
@@ -93,7 +98,8 @@ public class TestStandardFactories exten
*/
public void testKeywordTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Keyword").create(reader);
+ Tokenizer stream = tokenizerFactory("Keyword").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What's this thing do?" });
}
@@ -103,7 +109,8 @@ public class TestStandardFactories exten
*/
public void testWhitespaceTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Whitespace").create(reader);
+ Tokenizer stream = tokenizerFactory("Whitespace").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What's", "this", "thing", "do?" });
}
@@ -113,7 +120,8 @@ public class TestStandardFactories exten
*/
public void testLetterTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Letter").create(reader);
+ Tokenizer stream = tokenizerFactory("Letter").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What", "s", "this", "thing", "do" });
}
@@ -123,7 +131,8 @@ public class TestStandardFactories exten
*/
public void testLowerCaseTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("LowerCase").create(reader);
+ Tokenizer stream = tokenizerFactory("LowerCase").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "what", "s", "this", "thing", "do" });
}
@@ -133,7 +142,7 @@ public class TestStandardFactories exten
*/
public void testASCIIFolding() throws Exception {
Reader reader = new StringReader("Äeská");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ASCIIFolding").create(stream);
assertTokenStreamContents(stream, new String[] { "Ceska" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java Thu Jan 9 13:27:29 2014
@@ -20,7 +20,7 @@ package org.apache.lucene.analysis.stand
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -31,14 +31,16 @@ public class TestUAX29URLEmailTokenizerF
public void testUAX29URLEmailTokenizer() throws Exception {
Reader reader = new StringReader("Wha\u0301t's this thing do?");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Wha\u0301t's", "this", "thing", "do" });
}
public void testArabic() throws Exception {
Reader reader = new StringReader("اÙÙÙÙÙ
اÙÙثائÙ٠اÙØ£Ù٠ع٠ÙÙÙÙبÙدÙا ÙسÙ
Ù \"اÙØÙÙÙØ© باÙأرÙاÙ
: Ùصة ÙÙÙÙبÙدÙا\" (باÙØ¥ÙجÙÙزÙØ©: Truth in Numbers: The Wikipedia Story)Ø Ø³ÙتÙ
إطÙاÙÙ ÙÙ 2008.");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "اÙÙÙÙÙ
", "اÙÙثائÙÙ", "اÙØ£ÙÙ", "عÙ", "ÙÙÙÙبÙدÙا", "ÙسÙ
Ù", "اÙØÙÙÙØ©", "باÙأرÙاÙ
", "Ùصة", "ÙÙÙÙبÙدÙا",
"باÙØ¥ÙجÙÙزÙØ©", "Truth", "in", "Numbers", "The", "Wikipedia", "Story", "سÙتÙ
", "إطÙاÙÙ", "ÙÙ", "2008" });
@@ -46,21 +48,24 @@ public class TestUAX29URLEmailTokenizerF
public void testChinese() throws Exception {
Reader reader = new StringReader("ææ¯ä¸å½äººã ï¼ï¼ï¼ï¼ ï¼´ï½
ï½ï½ï½ ");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "æ", "æ¯", "ä¸", "å½", "人", "ï¼ï¼ï¼ï¼", "ï¼´ï½
ï½ï½ï½" });
}
public void testKorean() throws Exception {
Reader reader = new StringReader("ìë
íì¸ì íê¸ì
ëë¤");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "ìë
íì¸ì", "íê¸ì
ëë¤" });
}
public void testHyphen() throws Exception {
Reader reader = new StringReader("some-dashed-phrase");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "some", "dashed", "phrase" });
}
@@ -82,7 +87,8 @@ public class TestUAX29URLEmailTokenizerF
+ " blah Sirrah woof "
+ "http://[a42:a7b6::]/qSmxSUU4z/%52qVl4\n";
Reader reader = new StringReader(textWithURLs);
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {
"http://johno.jsmf.net/knowhow/ngrams/index.php?table=en-dickens-word-2gram¶graphs=50&length=200&no-ads=on",
@@ -120,7 +126,8 @@ public class TestUAX29URLEmailTokenizerF
+ "lMahAA.j/5.RqUjS745.DtkcYdi@d2-4gb-l6.ae\n"
+ "lv'p@tqk.vj5s0tgl.0dlu7su3iyiaz.dqso.494.3hb76.XN--MGBAAM7A8H\n";
Reader reader = new StringReader(textWithEmails);
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {
"some", "extra", "Words", "thrown", "in", "here",
@@ -149,8 +156,9 @@ public class TestUAX29URLEmailTokenizerF
String longWord = builder.toString();
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
- TokenStream stream = tokenizerFactory("UAX29URLEmail",
- "maxTokenLength", "1000").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail",
+ "maxTokenLength", "1000").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {"one", "two", "three", longWord, "four", "five", "six" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new SwedishLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@ public class TestSwedishLightStemFilter
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("jaktkarlens"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new SwedishLightStemFilter(sink));
}
@@ -70,8 +69,8 @@ public class TestSwedishLightStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SwedishLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.analysis.sv;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
@@ -30,7 +29,7 @@ import org.apache.lucene.analysis.util.B
public class TestSwedishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("äpplen äpple");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("SwedishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "äppl", "äppl" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestMultiWordSynonyms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestMultiWordSynonyms.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestMultiWordSynonyms.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestMultiWordSynonyms.java Thu Jan 9 13:27:29 2014
@@ -17,7 +17,6 @@ package org.apache.lucene.analysis.synon
* limitations under the License.
*/
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
@@ -32,7 +31,7 @@ public class TestMultiWordSynonyms exten
public void testMultiWordSynonyms() throws Exception {
Reader reader = new StringReader("a e");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Synonym", TEST_VERSION_CURRENT,
new StringMockResourceLoader("a b c,d"),
"synonyms", "synonyms.txt").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java Thu Jan 9 13:27:29 2014
@@ -49,8 +49,8 @@ public class TestSolrSynonymParser exten
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -122,8 +122,8 @@ public class TestSolrSynonymParser exten
final SynonymMap map = parser.build();
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -19,15 +19,10 @@ package org.apache.lucene.analysis.synon
import java.io.Reader;
import java.io.StringReader;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.pattern.PatternTokenizerFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
-import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
@@ -36,7 +31,7 @@ public class TestSynonymFilterFactory ex
/** checks for synonyms of "GB" in synonyms.txt */
private void checkSolrSynonyms(TokenFilterFactory factory) throws Exception {
Reader reader = new StringReader("GB");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTrue(stream instanceof SynonymFilter);
assertTokenStreamContents(stream,
@@ -47,7 +42,7 @@ public class TestSynonymFilterFactory ex
/** checks for synonyms of "second" in synonyms-wordnet.txt */
private void checkWordnetSynonyms(TokenFilterFactory factory) throws Exception {
Reader reader = new StringReader("second");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTrue(stream instanceof SynonymFilter);
assertTokenStreamContents(stream,
@@ -63,7 +58,7 @@ public class TestSynonymFilterFactory ex
/** if the synonyms are completely empty, test that we still analyze correctly */
public void testEmptySynonyms() throws Exception {
Reader reader = new StringReader("GB");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Synonym", TEST_VERSION_CURRENT,
new StringMockResourceLoader(""), // empty file!
"synonyms", "synonyms.txt").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java Thu Jan 9 13:27:29 2014
@@ -151,8 +151,8 @@ public class TestSynonymMapFilter extend
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
@@ -176,8 +176,8 @@ public class TestSynonymMapFilter extend
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
@@ -208,9 +208,8 @@ public class TestSynonymMapFilter extend
add("z", "boo", false);
add("y", "bee", true);
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
- true);
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -424,9 +423,9 @@ public class TestSynonymMapFilter extend
}
}
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE,
true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -496,8 +495,8 @@ public class TestSynonymMapFilter extend
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
}
};
@@ -525,7 +524,7 @@ public class TestSynonymMapFilter extend
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
return new TokenStreamComponents(tokenizer, new SynonymFilter(graph, map, ignoreCase));
@@ -552,8 +551,8 @@ public class TestSynonymMapFilter extend
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
TokenStream syns = new SynonymFilter(tokenizer, map, ignoreCase);
TokenStream graph = new MockGraphTokenFilter(random(), syns);
return new TokenStreamComponents(tokenizer, graph);
@@ -578,8 +577,8 @@ public class TestSynonymMapFilter extend
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
}
};
@@ -607,8 +606,8 @@ public class TestSynonymMapFilter extend
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
}
};
@@ -629,8 +628,8 @@ public class TestSynonymMapFilter extend
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -650,9 +649,9 @@ public class TestSynonymMapFilter extend
final boolean keepOrig = false;
add("aaa", "aaaa1 aaaa2 aaaa3", keepOrig);
add("bbb", "bbbb1 bbbb2", keepOrig);
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE,
true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -688,8 +687,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -728,8 +727,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -748,8 +747,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -766,8 +765,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -785,8 +784,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -802,9 +801,8 @@ public class TestSynonymMapFilter extend
final boolean keepOrig = false;
// b hangs off the end (no input token under it):
add("a", "a b", keepOrig);
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
- true);
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -835,8 +833,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -880,8 +878,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -899,8 +897,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -917,8 +915,8 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java Thu Jan 9 13:27:29 2014
@@ -46,8 +46,8 @@ public class TestWordnetSynonymParser ex
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Thu Jan 9 13:27:29 2014
@@ -131,8 +131,8 @@ public class TestThaiAnalyzer extends Ba
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ThaiWordFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiWordFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiWordFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiWordFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiWordFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -35,7 +35,7 @@ public class TestThaiWordFilterFactory e
public void testWordBreak() throws Exception {
assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
Reader reader = new StringReader("à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ThaiWord").create(stream);
assertTokenStreamContents(stream, new String[] {"à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹",
"à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ"});
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java Thu Jan 9 13:27:29 2014
@@ -37,8 +37,7 @@ public class TestTurkishLowerCaseFilter
* Test composed forms
*/
public void testTurkishLowerCaseFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0130STANBUL \u0130ZM\u0130R ISPARTA"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0130STANBUL \u0130ZM\u0130R ISPARTA");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"istanbul", "izmir",
"\u0131sparta",});
@@ -48,8 +47,7 @@ public class TestTurkishLowerCaseFilter
* Test decomposed forms
*/
public void testDecomposed() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0049\u0307STANBUL \u0049\u0307ZM\u0049\u0307R ISPARTA"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0049\u0307STANBUL \u0049\u0307ZM\u0049\u0307R ISPARTA");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"istanbul", "izmir",
"\u0131sparta",});
@@ -61,16 +59,14 @@ public class TestTurkishLowerCaseFilter
* to U+0130 + U+0316, and is lowercased the same way.
*/
public void testDecomposed2() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0049\u0316\u0307STANBUL \u0049\u0307ZM\u0049\u0307R I\u0316SPARTA"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0049\u0316\u0307STANBUL \u0049\u0307ZM\u0049\u0307R I\u0316SPARTA");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"i\u0316stanbul", "izmir",
"\u0131\u0316sparta",});
}
public void testDecomposed3() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0049\u0307"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0049\u0307");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"i"});
}
@@ -78,8 +74,8 @@ public class TestTurkishLowerCaseFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new TurkishLowerCaseFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@ public class TestTurkishLowerCaseFilterF
*/
public void testCasing() throws Exception {
Reader reader = new StringReader("AÄACI");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("TurkishLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "aÄacı" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Thu Jan 9 13:27:29 2014
@@ -53,7 +53,8 @@ public class TestCharTokenizers extends
}
// internal buffer size is 1024 make sure we have a surrogate pair right at the border
builder.insert(1023, "\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString()));
assertTokenStreamContents(tokenizer, builder.toString().toLowerCase(Locale.ROOT).split(" "));
}
@@ -70,7 +71,8 @@ public class TestCharTokenizers extends
builder.append("a");
}
builder.append("\ud801\udc1cabc");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT)});
}
}
@@ -84,7 +86,8 @@ public class TestCharTokenizers extends
for (int i = 0; i < 255; i++) {
builder.append("A");
}
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString() + builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
@@ -98,7 +101,8 @@ public class TestCharTokenizers extends
builder.append("A");
}
builder.append("\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString() + builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
@@ -106,8 +110,8 @@ public class TestCharTokenizers extends
public void testCrossPlaneNormalization() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT) {
@Override
protected int normalize(int c) {
if (c > 0xffff) {
@@ -144,8 +148,8 @@ public class TestCharTokenizers extends
public void testCrossPlaneNormalization2() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT) {
@Override
protected int normalize(int c) {
if (c <= 0xffff) {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java Thu Jan 9 13:27:29 2014
@@ -40,7 +40,8 @@ public class TestElision extends BaseTok
public void testElision() throws Exception {
String test = "Plop, juste pour voir l'embrouille avec O'brian. M'enfin.";
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(test));
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(test));
CharArraySet articles = new CharArraySet(TEST_VERSION_CURRENT, asSet("l", "M"), false);
TokenFilter filter = new ElisionFilter(tokenizer, articles);
List<String> tas = filter(filter);
@@ -64,8 +65,8 @@ public class TestElision extends BaseTok
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ElisionFilter(tokenizer, FrenchAnalyzer.DEFAULT_ARTICLES));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElisionFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElisionFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElisionFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElisionFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Simple tests to ensure the French elision filter factory is working.
@@ -32,7 +33,8 @@ public class TestElisionFilterFactory ex
*/
public void testElision() throws Exception {
Reader reader = new StringReader("l'avion");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("Elision", "articles", "frenchArticles.txt").create(stream);
assertTokenStreamContents(stream, new String[] { "avion" });
}
@@ -42,7 +44,8 @@ public class TestElisionFilterFactory ex
*/
public void testDefaultArticles() throws Exception {
Reader reader = new StringReader("l'avion");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("Elision").create(stream);
assertTokenStreamContents(stream, new String[] { "avion" });
}
@@ -52,7 +55,8 @@ public class TestElisionFilterFactory ex
*/
public void testCaseInsensitive() throws Exception {
Reader reader = new StringReader("L'avion");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("Elision",
"articles", "frenchArticles.txt",
"ignoreCase", "true").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,8 @@ import org.apache.lucene.analysis.wikipe
public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCase {
public void testTokenizer() throws Exception {
Reader reader = new StringReader("This is a [[Category:foo]]");
- Tokenizer tokenizer = tokenizerFactory("Wikipedia").create(reader);
+ Tokenizer tokenizer = tokenizerFactory("Wikipedia").create();
+ tokenizer.setReader(reader);
assertTokenStreamContents(tokenizer,
new String[] { "This", "is", "a", "foo" },
new int[] { 0, 5, 8, 21 },
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java Thu Jan 9 13:27:29 2014
@@ -18,7 +18,6 @@
package org.apache.lucene.analysis.wikipedia;
-import java.io.Reader;
import java.io.StringReader;
import java.io.IOException;
import java.util.Random;
@@ -40,7 +39,8 @@ public class WikipediaTokenizerTest exte
public void testSimple() throws Exception {
String text = "This is a [[Category:foo]]";
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(text));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(text));
assertTokenStreamContents(tf,
new String[] { "This", "is", "a", "foo" },
new int[] { 0, 5, 8, 21 },
@@ -62,7 +62,8 @@ public class WikipediaTokenizerTest exte
+ " [http://foo.boo.com/test/test/ Test Test] [http://foo.boo.com/test/test/test.html Test Test]"
+ " [http://foo.boo.com/test/test/test.html?g=b&c=d Test Test] <ref>Citation</ref> <sup>martian</sup> <span class=\"glue\">code</span>";
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] {"link", "This", "is", "a",
"foo", "Category", "This", "is", "a", "linked", "bar", "none",
@@ -103,7 +104,8 @@ public class WikipediaTokenizerTest exte
}
public void testLinkPhrases() throws Exception {
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(LINK_PHRASES));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(LINK_PHRASES));
checkLinkPhrases(tf);
}
@@ -116,7 +118,8 @@ public class WikipediaTokenizerTest exte
public void testLinks() throws Exception {
String test = "[http://lucene.apache.org/java/docs/index.html#news here] [http://lucene.apache.org/java/docs/index.html?b=c here] [https://lucene.apache.org/java/docs/index.html?b=c here]";
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "http://lucene.apache.org/java/docs/index.html#news", "here",
"http://lucene.apache.org/java/docs/index.html?b=c", "here",
@@ -131,10 +134,12 @@ public class WikipediaTokenizerTest exte
untoks.add(WikipediaTokenizer.CATEGORY);
untoks.add(WikipediaTokenizer.ITALICS);
//should be exactly the same, regardless of untoks
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(LINK_PHRASES), WikipediaTokenizer.TOKENS_ONLY, untoks);
+ WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.TOKENS_ONLY, untoks);
+ tf.setReader(new StringReader(LINK_PHRASES));
checkLinkPhrases(tf);
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
- tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.UNTOKENIZED_ONLY, untoks);
+ tf = new WikipediaTokenizer(WikipediaTokenizer.UNTOKENIZED_ONLY, untoks);
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "a b c d", "e f g", "link", "here", "link",
"there", "italics here", "something", "more italics", "h i j" },
@@ -150,7 +155,8 @@ public class WikipediaTokenizerTest exte
untoks.add(WikipediaTokenizer.ITALICS);
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
//should output all the indivual tokens plus the untokenized tokens as well. Untokenized tokens
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
+ WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "a b c d", "a", "b", "c", "d", "e f g", "e", "f", "g",
"link", "here", "link", "there", "italics here", "italics", "here",
@@ -161,7 +167,8 @@ public class WikipediaTokenizerTest exte
);
// now check the flags, TODO: add way to check flags from BaseTokenStreamTestCase?
- tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
+ tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+ tf.setReader(new StringReader(test));
int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0,
0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
FlagsAttribute flagsAtt = tf.addAttribute(FlagsAttribute.class);
@@ -179,8 +186,8 @@ public class WikipediaTokenizerTest exte
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new WikipediaTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new WikipediaTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -193,8 +200,8 @@ public class WikipediaTokenizerTest exte
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new WikipediaTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new WikipediaTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java Thu Jan 9 13:27:29 2014
@@ -64,11 +64,10 @@ public final class ICUTokenizer extends
* <p>
* The default attribute factory is used.
*
- * @param input Reader containing text to tokenize.
* @see DefaultICUTokenizerConfig
*/
- public ICUTokenizer(Reader input) {
- this(input, new DefaultICUTokenizerConfig(true));
+ public ICUTokenizer() {
+ this(new DefaultICUTokenizerConfig(true));
}
/**
@@ -77,11 +76,10 @@ public final class ICUTokenizer extends
* <p>
* The default attribute factory is used.
*
- * @param input Reader containing text to tokenize.
- * @param config Tailored BreakIterator configuration
+ * @param config Tailored BreakIterator configuration
*/
- public ICUTokenizer(Reader input, ICUTokenizerConfig config) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, config);
+ public ICUTokenizer(ICUTokenizerConfig config) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, config);
}
/**
@@ -89,11 +87,10 @@ public final class ICUTokenizer extends
* Reader, using a tailored BreakIterator configuration.
*
* @param factory AttributeFactory to use
- * @param input Reader containing text to tokenize.
- * @param config Tailored BreakIterator configuration
+ * @param config Tailored BreakIterator configuration
*/
- public ICUTokenizer(AttributeFactory factory, Reader input, ICUTokenizerConfig config) {
- super(factory, input);
+ public ICUTokenizer(AttributeFactory factory, ICUTokenizerConfig config) {
+ super(factory);
this.config = config;
breaker = new CompositeBreakIterator(config);
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java Thu Jan 9 13:27:29 2014
@@ -144,8 +144,8 @@ public class ICUTokenizerFactory extends
}
@Override
- public ICUTokenizer create(AttributeFactory factory, Reader input) {
+ public ICUTokenizer create(AttributeFactory factory) {
assert config != null : "inform must be called first!";
- return new ICUTokenizer(factory, input, config);
+ return new ICUTokenizer(factory, config);
}
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java Thu Jan 9 13:27:29 2014
@@ -81,8 +81,8 @@ public final class ICUCollationKeyAnalyz
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- KeywordTokenizer tokenizer = new KeywordTokenizer(factory, reader, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ KeywordTokenizer tokenizer = new KeywordTokenizer(factory, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
return new TokenStreamComponents(tokenizer, tokenizer);
}
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java Thu Jan 9 13:27:29 2014
@@ -29,8 +29,8 @@ import org.apache.lucene.analysis.core.K
public class TestICUFoldingFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
}
};
@@ -82,8 +82,8 @@ public class TestICUFoldingFilter extend
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -32,7 +32,7 @@ public class TestICUFoldingFilterFactory
public void test() throws Exception {
Reader reader = new StringReader("Résumé");
ICUFoldingFilterFactory factory = new ICUFoldingFilterFactory(new HashMap<String,String>());
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "resume" });
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java Thu Jan 9 13:27:29 2014
@@ -31,8 +31,8 @@ import com.ibm.icu.text.Normalizer2;
public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
}
};
@@ -61,8 +61,8 @@ public class TestICUNormalizer2Filter ex
public void testAlternate() throws IOException {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(
tokenizer,
/* specify nfc with decompose to get nfd */
@@ -82,8 +82,8 @@ public class TestICUNormalizer2Filter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java Thu Jan 9 13:27:29 2014
@@ -32,7 +32,7 @@ public class TestICUNormalizer2FilterFac
public void testDefaults() throws Exception {
Reader reader = new StringReader("This is a ï¼´ï½
ï½ï½");
ICUNormalizer2FilterFactory factory = new ICUNormalizer2FilterFactory(new HashMap<String,String>());
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "this", "is", "a", "test" });
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java Thu Jan 9 13:27:29 2014
@@ -66,7 +66,9 @@ public class TestICUTransformFilter exte
String rules = "a > b; b > c;"; // convert a's to b's and b's to c's
Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
assertTrue(custom.getFilter() == null);
- new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom);
+ final KeywordTokenizer input = new KeywordTokenizer();
+ input.setReader(new StringReader(""));
+ new ICUTransformFilter(input, custom);
assertTrue(custom.getFilter().equals(new UnicodeSet("[ab]")));
}
@@ -79,12 +81,16 @@ public class TestICUTransformFilter exte
String rules = "\\U00020087 > x;"; // convert CJK UNIFIED IDEOGRAPH-20087 to an x
Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
assertTrue(custom.getFilter() == null);
- new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom);
+ final KeywordTokenizer input = new KeywordTokenizer();
+ input.setReader(new StringReader(""));
+ new ICUTransformFilter(input, custom);
assertTrue(custom.getFilter().equals(new UnicodeSet("[\\U00020087]")));
}
private void checkToken(Transliterator transform, String input, String expected) throws IOException {
- TokenStream ts = new ICUTransformFilter(new KeywordTokenizer((new StringReader(input))), transform);
+ final KeywordTokenizer input1 = new KeywordTokenizer();
+ input1.setReader(new StringReader(input));
+ TokenStream ts = new ICUTransformFilter(input1, transform);
assertTokenStreamContents(ts, new String[] { expected });
}
@@ -93,8 +99,8 @@ public class TestICUTransformFilter exte
final Transliterator transform = Transliterator.getInstance("Any-Latin");
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, transform));
}
};
@@ -104,8 +110,8 @@ public class TestICUTransformFilter exte
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, Transliterator.getInstance("Any-Latin")));
}
};
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -35,7 +35,7 @@ public class TestICUTransformFilterFacto
Map<String,String> args = new HashMap<String,String>();
args.put("id", "Traditional-Simplified");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "ç®åå" });
}
@@ -47,7 +47,7 @@ public class TestICUTransformFilterFacto
Map<String,String> args = new HashMap<String,String>();
args.put("id", "Cyrillic-Latin");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "Rossijskaâ", "Federaciâ" });
}
@@ -59,7 +59,7 @@ public class TestICUTransformFilterFacto
args.put("id", "Cyrillic-Latin");
args.put("direction", "reverse");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "РоÑÑийÑкаÑ", "ФедеÑаÑиÑ" });
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Thu Jan 9 13:27:29 2014
@@ -42,7 +42,8 @@ public class TestICUTokenizer extends Ba
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input), new DefaultICUTokenizerConfig(false));
+ ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ tokenizer.setReader(new StringReader(input));
assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
@@ -52,7 +53,8 @@ public class TestICUTokenizer extends Ba
sb.append('a');
}
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input), new DefaultICUTokenizerConfig(false));
+ ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ tokenizer.setReader(new StringReader(input));
char token[] = new char[4096];
Arrays.fill(token, 'a');
String expectedToken = new String(token);
@@ -67,9 +69,8 @@ public class TestICUTokenizer extends Ba
private Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer tokenizer = new ICUTokenizer(reader, new DefaultICUTokenizerConfig(false));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
TokenFilter filter = new ICUNormalizer2Filter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java Thu Jan 9 13:27:29 2014
@@ -29,8 +29,8 @@ import org.apache.lucene.analysis.BaseTo
public class TestICUTokenizerCJK extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new ICUTokenizer(reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new ICUTokenizer());
}
};