You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/04 14:19:36 UTC
svn commit: r1592339 [1/2] - in /lucene/dev/trunk/lucene:
analysis/common/src/test/org/apache/lucene/analysis/ar/
analysis/common/src/test/org/apache/lucene/analysis/ckb/
analysis/common/src/test/org/apache/lucene/analysis/compound/
analysis/common/src...
Author: rmuir
Date: Sun May 4 12:19:35 2014
New Revision: 1592339
URL: http://svn.apache.org/r1592339
Log:
LUCENE-5642: Randomize attributeFactory in tests, use MockTokenizer more where possible, reduce use of esoteric Token ctors
Modified:
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java
lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java
lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java
lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java Sun May 4 12:19:35 2014
@@ -34,8 +34,7 @@ public class TestArabicFilters extends B
*/
public void testNormalizer() throws Exception {
Reader reader = new StringReader("اÙØ°ÙÙ Ù
ÙÙÙت Ø£ÙÙ
اÙÙÙ
");
- Tokenizer tokenizer = tokenizerFactory("Standard").create();
- tokenizer.setReader(reader);
+ Tokenizer tokenizer = whitespaceMockTokenizer(reader);
TokenStream stream = tokenFilterFactory("ArabicNormalization").create(tokenizer);
assertTokenStreamContents(stream, new String[] {"اÙØ°ÙÙ", "Ù
ÙÙت", "اÙÙ
اÙÙÙ
"});
}
@@ -45,8 +44,7 @@ public class TestArabicFilters extends B
*/
public void testStemmer() throws Exception {
Reader reader = new StringReader("اÙØ°ÙÙ Ù
ÙÙÙت Ø£ÙÙ
اÙÙÙ
");
- Tokenizer tokenizer = tokenizerFactory("Standard").create();
- tokenizer.setReader(reader);
+ Tokenizer tokenizer = whitespaceMockTokenizer(reader);
TokenStream stream = tokenFilterFactory("ArabicNormalization").create(tokenizer);
stream = tokenFilterFactory("ArabicStem").create(stream);
assertTokenStreamContents(stream, new String[] {"Ø°ÙÙ", "Ù
ÙÙت", "اÙÙ
اÙÙÙ
"});
@@ -57,8 +55,7 @@ public class TestArabicFilters extends B
*/
public void testPersianCharFilter() throws Exception {
Reader reader = charFilterFactory("Persian").create(new StringReader("Ù
ÛâØ®Ùرد"));
- Tokenizer tokenizer = tokenizerFactory("Standard").create();
- tokenizer.setReader(reader);
+ Tokenizer tokenizer = whitespaceMockTokenizer(reader);
assertTokenStreamContents(tokenizer, new String[] { "Ù
Û", "Ø®Ùرد" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java Sun May 4 12:19:35 2014
@@ -22,6 +22,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -32,7 +33,7 @@ public class TestSoraniNormalizationFilt
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new KeywordTokenizer();
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(tokenizer, new SoraniNormalizationFilter(tokenizer));
}
};
@@ -87,6 +88,13 @@ public class TestSoraniNormalizationFilt
}
public void testEmptyTerm() throws IOException {
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
+ return new TokenStreamComponents(tokenizer, new SoraniNormalizationFilter(tokenizer));
+ }
+ };
checkOneTerm(a, "", "");
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Sun May 4 12:19:35 2014
@@ -180,7 +180,7 @@ public class TestCompoundWordTokenFilter
public void testTokenEndingWithWordComponentOfMinimumLength() throws Exception {
CharArraySet dict = makeDictionary("ab", "cd", "ef");
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setReader(new StringReader("abcdef"));
DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
tokenizer,
@@ -200,7 +200,7 @@ public class TestCompoundWordTokenFilter
public void testWordComponentWithLessThanMinimumLength() throws Exception {
CharArraySet dict = makeDictionary("abc", "d", "efg");
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setReader(new StringReader("abcdefg"));
DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
tokenizer,
@@ -222,7 +222,8 @@ public class TestCompoundWordTokenFilter
CharArraySet dict = makeDictionary("Rind", "Fleisch", "Draht", "Schere", "Gesetz",
"Aufgabe", "Ãberwachung");
- Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ MockTokenizer wsTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ wsTokenizer.setEnableChecks(false); // we will reset in a strange place
wsTokenizer.setReader(new StringReader("Rindfleischüberwachungsgesetz"));
DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
wsTokenizer, dict,
@@ -246,7 +247,7 @@ public class TestCompoundWordTokenFilter
public void testRetainMockAttribute() throws Exception {
CharArraySet dict = makeDictionary("abc", "d", "efg");
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setReader(new StringReader("abcdefg"));
TokenStream stream = new MockRetainAttributeFilter(tokenizer);
stream = new DictionaryCompoundWordTokenFilter(
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java Sun May 4 12:19:35 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockReaderWrapper;
import org.apache.lucene.analysis.TokenStream;
@@ -29,7 +30,6 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.BasicOperations;
@@ -44,7 +44,7 @@ import org.apache.lucene.util.automaton.
* Any tests here need to probably consider unicode version of the JRE (it could
* cause false fails).
*/
-public class TestDuelingAnalyzers extends LuceneTestCase {
+public class TestDuelingAnalyzers extends BaseTokenStreamTestCase {
private CharacterRunAutomaton jvmLetter;
@Override
@@ -71,7 +71,7 @@ public class TestDuelingAnalyzers extend
Analyzer right = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -91,7 +91,7 @@ public class TestDuelingAnalyzers extend
Analyzer right = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -109,7 +109,7 @@ public class TestDuelingAnalyzers extend
Analyzer right = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -128,7 +128,7 @@ public class TestDuelingAnalyzers extend
Analyzer right = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -146,7 +146,7 @@ public class TestDuelingAnalyzers extend
Analyzer right = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -165,7 +165,7 @@ public class TestDuelingAnalyzers extend
Analyzer right = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java Sun May 4 12:19:35 2014
@@ -179,7 +179,7 @@ public class TestFactories extends BaseT
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tf = tokenizer.create();
+ Tokenizer tf = tokenizer.create(newAttributeFactory());
if (tokenfilter != null) {
return new TokenStreamComponents(tf, tokenfilter.create(tf));
} else {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Sun May 4 12:19:35 2014
@@ -355,6 +355,11 @@ public class TestRandomChains extends Ba
return TEST_VERSION_CURRENT;
}
});
+ put(AttributeFactory.class, new ArgProducer() {
+ @Override public Object create(Random random) {
+ return newAttributeFactory(random);
+ }
+ });
put(Set.class, new ArgProducer() {
@Override public Object create(Random random) {
// TypeTokenFilter
@@ -582,10 +587,7 @@ public class TestRandomChains extends Ba
Object[] args = new Object[paramTypes.length];
for (int i = 0; i < args.length; i++) {
Class<?> paramType = paramTypes[i];
- if (paramType == AttributeFactory.class) {
- // TODO: maybe the collator one...???
- args[i] = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
- } else if (paramType == AttributeSource.class) {
+ if (paramType == AttributeSource.class) {
// TODO: args[i] = new AttributeSource();
// this is currently too scary to deal with!
args[i] = null; // force IAE
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java Sun May 4 12:19:35 2014
@@ -50,7 +50,7 @@ public class TestStandardAnalyzer extend
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer);
}
};
@@ -298,7 +298,7 @@ public class TestStandardAnalyzer extend
new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
TokenStream tokenStream = new MockGraphTokenFilter(random(), tokenizer);
return new TokenStreamComponents(tokenizer, tokenStream);
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java Sun May 4 12:19:35 2014
@@ -36,7 +36,7 @@ public class TestTypeTokenFilter extends
public void testTypeFilter() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set<String> stopTypes = asSet("<NUM>");
- final StandardTokenizer input = new StandardTokenizer(TEST_VERSION_CURRENT);
+ final StandardTokenizer input = new StandardTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
input.setReader(reader);
TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, input, stopTypes);
assertTokenStreamContents(stream, new String[]{"is", "palindrome", "while", "is", "not"});
@@ -85,7 +85,7 @@ public class TestTypeTokenFilter extends
public void testTypeFilterWhitelist() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set<String> stopTypes = Collections.singleton("<NUM>");
- final StandardTokenizer input = new StandardTokenizer(TEST_VERSION_CURRENT);
+ final StandardTokenizer input = new StandardTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
input.setReader(reader);
TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, input, stopTypes, true);
assertTokenStreamContents(stream, new String[]{"121", "123"});
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java Sun May 4 12:19:35 2014
@@ -47,7 +47,7 @@ public class TestUAX29URLEmailTokenizer
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
tokenizer.setReader(new StringReader(input));
BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
@@ -56,7 +56,7 @@ public class TestUAX29URLEmailTokenizer
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
return new TokenStreamComponents(tokenizer);
}
};
@@ -103,7 +103,7 @@ public class TestUAX29URLEmailTokenizer
private Analyzer urlAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
tokenizer.setMaxTokenLength(Integer.MAX_VALUE); // Tokenize arbitrary length URLs
TokenFilter filter = new URLFilter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
@@ -113,7 +113,7 @@ public class TestUAX29URLEmailTokenizer
private Analyzer emailAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
TokenFilter filter = new EmailFilter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Sun May 4 12:19:35 2014
@@ -42,7 +42,7 @@ public class TestGermanStemFilter extend
Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer t = new KeywordTokenizer();
+ Tokenizer t = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(t,
new GermanStemFilter(new LowerCaseFilter(TEST_VERSION_CURRENT, t)));
}
@@ -54,12 +54,6 @@ public class TestGermanStemFilter extend
vocOut.close();
}
- // LUCENE-3043: we use keywordtokenizer in this test,
- // so ensure the stemmer does not crash on zero-length strings.
- public void testEmpty() throws Exception {
- assertAnalyzesTo(analyzer, "", new String[] { "" });
- }
-
public void testKeyword() throws IOException {
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
Analyzer a = new Analyzer() {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java Sun May 4 12:19:35 2014
@@ -24,6 +24,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -37,9 +38,8 @@ public class TestGalicianStemFilter exte
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
- TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
- return new TokenStreamComponents(source, new GalicianStemFilter(result));
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new GalicianStemFilter(source));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java Sun May 4 12:19:35 2014
@@ -34,8 +34,7 @@ public class TestHindiFilters extends Ba
*/
public void testIndicNormalizer() throws Exception {
Reader reader = new StringReader("তà§â à¤
ाà¥à¤°");
- TokenStream stream = tokenizerFactory("Standard").create();
- ((Tokenizer)stream).setReader(reader);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "à§", "à¤à¤°" });
}
@@ -45,8 +44,7 @@ public class TestHindiFilters extends Ba
*/
public void testHindiNormalizer() throws Exception {
Reader reader = new StringReader("à¥à¤¿à¤¤à¤¾à¤¬");
- TokenStream stream = tokenizerFactory("Standard").create();
- ((Tokenizer)stream).setReader(reader);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
assertTokenStreamContents(stream, new String[] {"à¤à¤¿à¤¤à¤¾à¤¬"});
@@ -57,8 +55,7 @@ public class TestHindiFilters extends Ba
*/
public void testStemmer() throws Exception {
Reader reader = new StringReader("à¤à¤¿à¤¤à¤¾à¤¬à¥à¤");
- TokenStream stream = tokenizerFactory("Standard").create();
- ((Tokenizer)stream).setReader(reader);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
stream = tokenFilterFactory("HindiStem").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java Sun May 4 12:19:35 2014
@@ -22,6 +22,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -33,7 +34,7 @@ public class TestIndonesianStemmer exten
Analyzer a = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new KeywordTokenizer();
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
}
};
@@ -114,7 +115,7 @@ public class TestIndonesianStemmer exten
Analyzer b = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new KeywordTokenizer();
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer, false));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java Sun May 4 12:19:35 2014
@@ -37,7 +37,8 @@ public class TestSingleTokenTokenFilter
assertEquals(token, tokenAtt);
assertFalse(ts.incrementToken());
- token = new Token("hallo", 10, 20, "someType");
+ token = new Token("hallo", 10, 20);
+ token.setType("someType");
ts.setToken(token);
ts.reset();
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java Sun May 4 12:19:35 2014
@@ -46,11 +46,11 @@ public class TestTrimFilter extends Base
char[] whitespace = " ".toCharArray();
char[] empty = "".toCharArray();
- TokenStream ts = new IterTokenStream(new Token(a, 0, a.length, 1, 5),
- new Token(b, 0, b.length, 6, 10),
- new Token(ccc, 0, ccc.length, 11, 15),
- new Token(whitespace, 0, whitespace.length, 16, 20),
- new Token(empty, 0, empty.length, 21, 21));
+ TokenStream ts = new IterTokenStream(new Token(new String(a, 0, a.length), 1, 5),
+ new Token(new String(b, 0, b.length), 6, 10),
+ new Token(new String(ccc, 0, ccc.length), 11, 15),
+ new Token(new String(whitespace, 0, whitespace.length), 16, 20),
+ new Token(new String(empty, 0, empty.length), 21, 21));
ts = new TrimFilter(TEST_VERSION_CURRENT, ts);
assertTokenStreamContents(ts, new String[] { "a", "b", "cCc", "", ""});
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java Sun May 4 12:19:35 2014
@@ -355,7 +355,8 @@ public class TestWordDelimiterFilter ext
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
}
};
- checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
}
}
@@ -379,7 +380,8 @@ public class TestWordDelimiterFilter ext
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
}
};
- checkRandomData(random(), a, 100*RANDOM_MULTIPLIER, 8192);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random(), a, 100*RANDOM_MULTIPLIER, 8192, false, false);
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java Sun May 4 12:19:35 2014
@@ -27,11 +27,14 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
+import static org.apache.lucene.analysis.path.PathHierarchyTokenizer.DEFAULT_DELIMITER;
+import static org.apache.lucene.analysis.path.PathHierarchyTokenizer.DEFAULT_SKIP;
+
public class TestPathHierarchyTokenizer extends BaseTokenStreamTestCase {
public void testBasic() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a", "/a/b", "/a/b/c"},
@@ -43,7 +46,7 @@ public class TestPathHierarchyTokenizer
public void testEndOfDelimiter() throws Exception {
String path = "/a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"/a", "/a/b", "/a/b/c", "/a/b/c/"},
@@ -55,7 +58,7 @@ public class TestPathHierarchyTokenizer
public void testStartOfChar() throws Exception {
String path = "a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"a", "a/b", "a/b/c"},
@@ -67,7 +70,7 @@ public class TestPathHierarchyTokenizer
public void testStartOfCharEndOfDelimiter() throws Exception {
String path = "a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"a", "a/b", "a/b/c", "a/b/c/"},
@@ -79,7 +82,7 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimiter() throws Exception {
String path = "/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"/"},
@@ -91,7 +94,7 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimiters() throws Exception {
String path = "//";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/", "//"},
@@ -103,7 +106,7 @@ public class TestPathHierarchyTokenizer
public void testReplace() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( '/', '\\' );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), '/', '\\', DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"\\a", "\\a\\b", "\\a\\b\\c"},
@@ -115,7 +118,7 @@ public class TestPathHierarchyTokenizer
public void testWindowsPath() throws Exception {
String path = "c:\\a\\b\\c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( '\\', '\\' );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), '\\', '\\', DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"c:", "c:\\a", "c:\\a\\b", "c:\\a\\b\\c"},
@@ -131,7 +134,7 @@ public class TestPathHierarchyTokenizer
NormalizeCharMap normMap = builder.build();
String path = "c:\\a\\b\\c";
Reader cs = new MappingCharFilter(normMap, new StringReader(path));
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(cs);
assertTokenStreamContents(t,
new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
@@ -143,7 +146,7 @@ public class TestPathHierarchyTokenizer
public void testBasicSkip() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c"},
@@ -155,7 +158,7 @@ public class TestPathHierarchyTokenizer
public void testEndOfDelimiterSkip() throws Exception {
String path = "/a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c", "/b/c/"},
@@ -167,7 +170,7 @@ public class TestPathHierarchyTokenizer
public void testStartOfCharSkip() throws Exception {
String path = "a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c"},
@@ -179,7 +182,7 @@ public class TestPathHierarchyTokenizer
public void testStartOfCharEndOfDelimiterSkip() throws Exception {
String path = "a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer(1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c", "/b/c/"},
@@ -191,7 +194,7 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimiterSkip() throws Exception {
String path = "/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{},
@@ -203,7 +206,7 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimitersSkip() throws Exception {
String path = "//";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader( new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
@@ -218,11 +221,12 @@ public class TestPathHierarchyTokenizer
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new PathHierarchyTokenizer();
+ Tokenizer tokenizer = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
}
/** blast some random large strings through the analyzer */
@@ -231,10 +235,11 @@ public class TestPathHierarchyTokenizer
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new PathHierarchyTokenizer();
+ Tokenizer tokenizer = new PathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027, false, false);
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java Sun May 4 12:19:35 2014
@@ -26,11 +26,14 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
+import static org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer.DEFAULT_DELIMITER;
+import static org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer.DEFAULT_SKIP;
+
public class TestReversePathHierarchyTokenizer extends BaseTokenStreamTestCase {
public void testBasicReverse() throws Exception {
String path = "/a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/b/c", "a/b/c", "b/c", "c"},
@@ -42,7 +45,7 @@ public class TestReversePathHierarchyTok
public void testEndOfDelimiterReverse() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/b/c/", "a/b/c/", "b/c/", "c/"},
@@ -54,7 +57,7 @@ public class TestReversePathHierarchyTok
public void testStartOfCharReverse() throws Exception {
String path = "a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/c", "b/c", "c"},
@@ -66,7 +69,7 @@ public class TestReversePathHierarchyTok
public void testStartOfCharEndOfDelimiterReverse() throws Exception {
String path = "a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/c/", "b/c/", "c/"},
@@ -78,7 +81,7 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimiterReverse() throws Exception {
String path = "/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
@@ -90,7 +93,7 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimitersReverse() throws Exception {
String path = "//";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"//", "/"},
@@ -102,7 +105,7 @@ public class TestReversePathHierarchyTok
public void testEndOfDelimiterReverseSkip() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
new StringReader(path);
assertTokenStreamContents(t,
@@ -115,7 +118,7 @@ public class TestReversePathHierarchyTok
public void testStartOfCharReverseSkip() throws Exception {
String path = "a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/", "b/"},
@@ -127,7 +130,7 @@ public class TestReversePathHierarchyTok
public void testStartOfCharEndOfDelimiterReverseSkip() throws Exception {
String path = "a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/", "b/"},
@@ -139,7 +142,7 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimiterReverseSkip() throws Exception {
String path = "/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{},
@@ -151,7 +154,7 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimitersReverseSkip() throws Exception {
String path = "//";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 1);
t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
@@ -163,7 +166,7 @@ public class TestReversePathHierarchyTok
public void testReverseSkip2() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 2 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, 2);
t.setReader( new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/", "a/"},
@@ -178,11 +181,12 @@ public class TestReversePathHierarchyTok
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new ReversePathHierarchyTokenizer();
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
}
/** blast some random large strings through the analyzer */
@@ -191,10 +195,11 @@ public class TestReversePathHierarchyTok
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new ReversePathHierarchyTokenizer();
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer(newAttributeFactory(), DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 1027, false, false);
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java Sun May 4 12:19:35 2014
@@ -53,7 +53,7 @@ public class TestPatternTokenizer extend
};
for( String[] test : tests ) {
- TokenStream stream = new PatternTokenizer(Pattern.compile(test[1]), Integer.parseInt(test[0]));
+ TokenStream stream = new PatternTokenizer(newAttributeFactory(), Pattern.compile(test[1]), Integer.parseInt(test[0]));
((Tokenizer)stream).setReader(new StringReader(test[2]));
String out = tsToString( stream );
// System.out.println( test[2] + " ==> " + out );
@@ -86,7 +86,7 @@ public class TestPatternTokenizer extend
CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
// create PatternTokenizer
- Tokenizer stream = new PatternTokenizer(Pattern.compile("[,;/\\s]+"), -1);
+ Tokenizer stream = new PatternTokenizer(newAttributeFactory(), Pattern.compile("[,;/\\s]+"), -1);
stream.setReader(charStream);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther", "is", "here" },
@@ -95,7 +95,7 @@ public class TestPatternTokenizer extend
INPUT.length());
charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
- stream = new PatternTokenizer(Pattern.compile("Günther"), 0);
+ stream = new PatternTokenizer(newAttributeFactory(), Pattern.compile("Günther"), 0);
stream.setReader(charStream);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther" },
@@ -132,7 +132,7 @@ public class TestPatternTokenizer extend
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new PatternTokenizer(Pattern.compile("a"), -1);
+ Tokenizer tokenizer = new PatternTokenizer(newAttributeFactory(), Pattern.compile("a"), -1);
return new TokenStreamComponents(tokenizer);
}
};
@@ -141,7 +141,7 @@ public class TestPatternTokenizer extend
Analyzer b = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new PatternTokenizer(Pattern.compile("a"), 0);
+ Tokenizer tokenizer = new PatternTokenizer(newAttributeFactory(), Pattern.compile("a"), 0);
return new TokenStreamComponents(tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java Sun May 4 12:19:35 2014
@@ -28,7 +28,7 @@ public class TestPatternTokenizerFactory
public void testFactory() throws Exception {
final Reader reader = new StringReader("Günther Günther is here");
// create PatternTokenizer
- Tokenizer stream = tokenizerFactory("Pattern", "pattern", "[,;/\\s]+").create();
+ Tokenizer stream = tokenizerFactory("Pattern", "pattern", "[,;/\\s]+").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther", "is", "here" });
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java Sun May 4 12:19:35 2014
@@ -18,18 +18,14 @@ package org.apache.lucene.analysis.pt;
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -41,9 +37,8 @@ public class TestPortugueseLightStemFilt
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
- TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
- return new TokenStreamComponents(source, new PortugueseLightStemFilter(result));
+ Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(source, new PortugueseLightStemFilter(source));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java Sun May 4 12:19:35 2014
@@ -18,18 +18,14 @@ package org.apache.lucene.analysis.pt;
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -41,9 +37,8 @@ public class TestPortugueseMinimalStemFi
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
- TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
- return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(result));
+ Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(source));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java Sun May 4 12:19:35 2014
@@ -20,7 +20,6 @@ package org.apache.lucene.analysis.pt;
import static org.apache.lucene.analysis.VocabularyAssert.assertVocabulary;
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -28,9 +27,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
/**
@@ -40,9 +37,8 @@ public class TestPortugueseStemFilter ex
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
- TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
- return new TokenStreamComponents(source, new PortugueseStemFilter(result));
+ Tokenizer source = new MockTokenizer(MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(source, new PortugueseStemFilter(source));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java Sun May 4 12:19:35 2014
@@ -1096,7 +1096,8 @@ public class ShingleFilterTest extends B
private static Token createToken
(String term, int start, int offset, int positionIncrement)
{
- Token token = new Token(start, offset);
+ Token token = new Token();
+ token.setOffset(start, offset);
token.copyBuffer(term.toCharArray(), 0, term.length());
token.setPositionIncrement(positionIncrement);
return token;
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java Sun May 4 12:19:35 2014
@@ -151,7 +151,9 @@ public class TestTeeSinkTokenFilter exte
dogDetector.addAttribute(CheckClearAttributesAttribute.class);
theDetector.addAttribute(CheckClearAttributesAttribute.class);
- final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(whitespaceMockTokenizer(buffer2.toString()));
+ MockTokenizer tokenizer = new MockTokenizer(tee1.getAttributeFactory(), MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader(buffer2.toString()));
+ final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(tokenizer);
tee2.addSinkTokenStream(dogDetector);
tee2.addSinkTokenStream(theDetector);
final TokenStream source2 = tee2;
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java Sun May 4 12:19:35 2014
@@ -34,7 +34,7 @@ public class TestStandardFactories exten
*/
public void testStandardTokenizer() throws Exception {
Reader reader = new StringReader("Wha\u0301t's this thing do?");
- Tokenizer stream = tokenizerFactory("Standard").create();
+ Tokenizer stream = tokenizerFactory("Standard").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[]{"Wha\u0301t's", "this", "thing", "do"});
@@ -49,7 +49,7 @@ public class TestStandardFactories exten
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
Tokenizer stream = tokenizerFactory("Standard",
- "maxTokenLength", "1000").create();
+ "maxTokenLength", "1000").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[]{"one", "two", "three", longWord, "four", "five", "six"});
@@ -60,7 +60,7 @@ public class TestStandardFactories exten
*/
public void testClassicTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- Tokenizer stream = tokenizerFactory("Classic").create();
+ Tokenizer stream = tokenizerFactory("Classic").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[]{"What's", "this", "thing", "do"});
@@ -75,7 +75,7 @@ public class TestStandardFactories exten
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
Tokenizer stream = tokenizerFactory("Classic",
- "maxTokenLength", "1000").create();
+ "maxTokenLength", "1000").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[]{"one", "two", "three", longWord, "four", "five", "six"});
@@ -86,7 +86,7 @@ public class TestStandardFactories exten
*/
public void testStandardFilter() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- Tokenizer tokenizer = tokenizerFactory("Classic").create();
+ Tokenizer tokenizer = tokenizerFactory("Classic").create(newAttributeFactory());
tokenizer.setReader(reader);
TokenStream stream = tokenFilterFactory("Classic").create(tokenizer);
assertTokenStreamContents(stream,
@@ -109,7 +109,7 @@ public class TestStandardFactories exten
*/
public void testWhitespaceTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- Tokenizer stream = tokenizerFactory("Whitespace").create();
+ Tokenizer stream = tokenizerFactory("Whitespace").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What's", "this", "thing", "do?" });
@@ -120,7 +120,7 @@ public class TestStandardFactories exten
*/
public void testLetterTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- Tokenizer stream = tokenizerFactory("Letter").create();
+ Tokenizer stream = tokenizerFactory("Letter").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What", "s", "this", "thing", "do" });
@@ -131,7 +131,7 @@ public class TestStandardFactories exten
*/
public void testLowerCaseTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- Tokenizer stream = tokenizerFactory("LowerCase").create();
+ Tokenizer stream = tokenizerFactory("LowerCase").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "what", "s", "this", "thing", "do" });
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java Sun May 4 12:19:35 2014
@@ -31,7 +31,7 @@ public class TestUAX29URLEmailTokenizerF
public void testUAX29URLEmailTokenizer() throws Exception {
Reader reader = new StringReader("Wha\u0301t's this thing do?");
- Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Wha\u0301t's", "this", "thing", "do" });
@@ -39,7 +39,7 @@ public class TestUAX29URLEmailTokenizerF
public void testArabic() throws Exception {
Reader reader = new StringReader("اÙÙÙÙÙ
اÙÙثائÙ٠اÙØ£Ù٠ع٠ÙÙÙÙبÙدÙا ÙسÙ
Ù \"اÙØÙÙÙØ© باÙأرÙاÙ
: Ùصة ÙÙÙÙبÙدÙا\" (باÙØ¥ÙجÙÙزÙØ©: Truth in Numbers: The Wikipedia Story)Ø Ø³ÙتÙ
إطÙاÙÙ ÙÙ 2008.");
- Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "اÙÙÙÙÙ
", "اÙÙثائÙÙ", "اÙØ£ÙÙ", "عÙ", "ÙÙÙÙبÙدÙا", "ÙسÙ
Ù", "اÙØÙÙÙØ©", "باÙأرÙاÙ
", "Ùصة", "ÙÙÙÙبÙدÙا",
@@ -48,7 +48,7 @@ public class TestUAX29URLEmailTokenizerF
public void testChinese() throws Exception {
Reader reader = new StringReader("ææ¯ä¸å½äººã ï¼ï¼ï¼ï¼ ï¼´ï½
ï½ï½ï½ ");
- Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "æ", "æ¯", "ä¸", "å½", "人", "ï¼ï¼ï¼ï¼", "ï¼´ï½
ï½ï½ï½" });
@@ -56,7 +56,7 @@ public class TestUAX29URLEmailTokenizerF
public void testKorean() throws Exception {
Reader reader = new StringReader("ìë
íì¸ì íê¸ì
ëë¤");
- Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "ìë
íì¸ì", "íê¸ì
ëë¤" });
@@ -64,7 +64,7 @@ public class TestUAX29URLEmailTokenizerF
public void testHyphen() throws Exception {
Reader reader = new StringReader("some-dashed-phrase");
- Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "some", "dashed", "phrase" });
@@ -87,7 +87,7 @@ public class TestUAX29URLEmailTokenizerF
+ " blah Sirrah woof "
+ "http://[a42:a7b6::]/qSmxSUU4z/%52qVl4\n";
Reader reader = new StringReader(textWithURLs);
- Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {
@@ -126,7 +126,7 @@ public class TestUAX29URLEmailTokenizerF
+ "lMahAA.j/5.RqUjS745.DtkcYdi@d2-4gb-l6.ae\n"
+ "lv'p@tqk.vj5s0tgl.0dlu7su3iyiaz.dqso.494.3hb76.XN--MGBAAM7A8H\n";
Reader reader = new StringReader(textWithEmails);
- Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {
@@ -157,7 +157,7 @@ public class TestUAX29URLEmailTokenizerF
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
Tokenizer stream = tokenizerFactory("UAX29URLEmail",
- "maxTokenLength", "1000").create();
+ "maxTokenLength", "1000").create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {"one", "two", "three", longWord, "four", "five", "six" });
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java Sun May 4 12:19:35 2014
@@ -31,7 +31,7 @@ public class TestThaiTokenizerFactory ex
*/
public void testWordBreak() throws Exception {
assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiTokenizer.DBBI_AVAILABLE);
- Tokenizer tokenizer = tokenizerFactory("Thai").create();
+ Tokenizer tokenizer = tokenizerFactory("Thai").create(newAttributeFactory());
tokenizer.setReader(new StringReader("à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ"));
assertTokenStreamContents(tokenizer, new String[] {"à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹",
"à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ"});
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Sun May 4 12:19:35 2014
@@ -52,7 +52,7 @@ public class TestCharTokenizers extends
}
// internal buffer size is 1024 make sure we have a surrogate pair right at the border
builder.insert(1023, "\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString()));
assertTokenStreamContents(tokenizer, builder.toString().toLowerCase(Locale.ROOT).split(" "));
}
@@ -70,7 +70,7 @@ public class TestCharTokenizers extends
builder.append("a");
}
builder.append("\ud801\udc1cabc");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT)});
}
@@ -85,7 +85,7 @@ public class TestCharTokenizers extends
for (int i = 0; i < 255; i++) {
builder.append("A");
}
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
@@ -100,7 +100,7 @@ public class TestCharTokenizers extends
builder.append("A");
}
builder.append("\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
@@ -110,7 +110,7 @@ public class TestCharTokenizers extends
Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory()) {
@Override
protected int normalize(int c) {
if (c > 0xffff) {
@@ -148,7 +148,7 @@ public class TestCharTokenizers extends
Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, newAttributeFactory()) {
@Override
protected int normalize(int c) {
if (c <= 0xffff) {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java Sun May 4 12:19:35 2014
@@ -40,7 +40,7 @@ public class TestElision extends BaseTok
public void testElision() throws Exception {
String test = "Plop, juste pour voir l'embrouille avec O'brian. M'enfin.";
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, newAttributeFactory());
tokenizer.setReader(new StringReader(test));
CharArraySet articles = new CharArraySet(TEST_VERSION_CURRENT, asSet("l", "M"), false);
TokenFilter filter = new ElisionFilter(tokenizer, articles);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java Sun May 4 12:19:35 2014
@@ -140,7 +140,7 @@ public class TestSegmentingTokenizerBase
private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
public WholeSentenceTokenizer() {
- super(BreakIterator.getSentenceInstance(Locale.ROOT));
+ super(newAttributeFactory(), BreakIterator.getSentenceInstance(Locale.ROOT));
}
@Override
@@ -178,7 +178,7 @@ public class TestSegmentingTokenizerBase
private PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
public SentenceAndWordTokenizer() {
- super(BreakIterator.getSentenceInstance(Locale.ROOT));
+ super(newAttributeFactory(), BreakIterator.getSentenceInstance(Locale.ROOT));
}
@Override
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java Sun May 4 12:19:35 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.wikipe
public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCase {
public void testTokenizer() throws Exception {
Reader reader = new StringReader("This is a [[Category:foo]]");
- Tokenizer tokenizer = tokenizerFactory("Wikipedia").create();
+ Tokenizer tokenizer = tokenizerFactory("Wikipedia").create(newAttributeFactory());
tokenizer.setReader(reader);
assertTokenStreamContents(tokenizer,
new String[] { "This", "is", "a", "foo" },