You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/04 14:19:36 UTC
svn commit: r1592339 [2/2] - in /lucene/dev/trunk/lucene:
analysis/common/src/test/org/apache/lucene/analysis/ar/
analysis/common/src/test/org/apache/lucene/analysis/ckb/
analysis/common/src/test/org/apache/lucene/analysis/compound/
analysis/common/src...
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java Sun May 4 12:19:35 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.wikip
import java.io.StringReader;
import java.io.IOException;
+import java.util.Collections;
import java.util.Random;
import java.util.Set;
import java.util.HashSet;
@@ -39,7 +40,7 @@ public class WikipediaTokenizerTest exte
public void testSimple() throws Exception {
String text = "This is a [[Category:foo]]";
- WikipediaTokenizer tf = new WikipediaTokenizer();
+ WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
tf.setReader(new StringReader(text));
assertTokenStreamContents(tf,
new String[] { "This", "is", "a", "foo" },
@@ -62,7 +63,7 @@ public class WikipediaTokenizerTest exte
+ " [http://foo.boo.com/test/test/ Test Test] [http://foo.boo.com/test/test/test.html Test Test]"
+ " [http://foo.boo.com/test/test/test.html?g=b&c=d Test Test] <ref>Citation</ref> <sup>martian</sup> <span class=\"glue\">code</span>";
- WikipediaTokenizer tf = new WikipediaTokenizer();
+ WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] {"link", "This", "is", "a",
@@ -104,7 +105,7 @@ public class WikipediaTokenizerTest exte
}
public void testLinkPhrases() throws Exception {
- WikipediaTokenizer tf = new WikipediaTokenizer();
+ WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
tf.setReader(new StringReader(LINK_PHRASES));
checkLinkPhrases(tf);
}
@@ -118,7 +119,7 @@ public class WikipediaTokenizerTest exte
public void testLinks() throws Exception {
String test = "[http://lucene.apache.org/java/docs/index.html#news here] [http://lucene.apache.org/java/docs/index.html?b=c here] [https://lucene.apache.org/java/docs/index.html?b=c here]";
- WikipediaTokenizer tf = new WikipediaTokenizer();
+ WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "http://lucene.apache.org/java/docs/index.html#news", "here",
@@ -134,7 +135,7 @@ public class WikipediaTokenizerTest exte
untoks.add(WikipediaTokenizer.CATEGORY);
untoks.add(WikipediaTokenizer.ITALICS);
//should be exactly the same, regardless of untoks
- WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.TOKENS_ONLY, untoks);
+ WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, untoks);
tf.setReader(new StringReader(LINK_PHRASES));
checkLinkPhrases(tf);
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
@@ -155,7 +156,7 @@ public class WikipediaTokenizerTest exte
untoks.add(WikipediaTokenizer.ITALICS);
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
//should output all the indivual tokens plus the untokenized tokens as well. Untokenized tokens
- WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+ WikipediaTokenizer tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.BOTH, untoks);
tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "a b c d", "a", "b", "c", "d", "e f g", "e", "f", "g",
@@ -167,7 +168,7 @@ public class WikipediaTokenizerTest exte
);
// now check the flags, TODO: add way to check flags from BaseTokenStreamTestCase?
- tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+ tf = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.BOTH, untoks);
tf.setReader(new StringReader(test));
int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0,
0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
@@ -187,11 +188,12 @@ public class WikipediaTokenizerTest exte
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new WikipediaTokenizer();
+ Tokenizer tokenizer = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
}
/** blast some random large strings through the analyzer */
@@ -201,10 +203,11 @@ public class WikipediaTokenizerTest exte
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new WikipediaTokenizer();
+ Tokenizer tokenizer = new WikipediaTokenizer(newAttributeFactory(), WikipediaTokenizer.TOKENS_ONLY, Collections.<String>emptySet());
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
- checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 8192);
+ // TODO: properly support positionLengthAttribute
+ checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 8192, false, false);
}
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java Sun May 4 12:19:35 2014
@@ -77,7 +77,7 @@ public class TestICUNormalizer2CharFilte
CharFilter reader = new ICUNormalizer2CharFilter(new StringReader(input),
Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE));
- Tokenizer tokenStream = new NGramTokenizer(TEST_VERSION_CURRENT, 1, 1);
+ Tokenizer tokenStream = new NGramTokenizer(TEST_VERSION_CURRENT, newAttributeFactory(), 1, 1);
tokenStream.setReader(reader);
assertTokenStreamContents(tokenStream,
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Sun May 4 12:19:35 2014
@@ -42,7 +42,7 @@ public class TestICUTokenizer extends Ba
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
tokenizer.setReader(new StringReader(input));
assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
@@ -53,7 +53,7 @@ public class TestICUTokenizer extends Ba
sb.append('a');
}
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
tokenizer.setReader(new StringReader(input));
char token[] = new char[4096];
Arrays.fill(token, 'a');
@@ -70,7 +70,7 @@ public class TestICUTokenizer extends Ba
private Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
TokenFilter filter = new ICUNormalizer2Filter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
}
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java Sun May 4 12:19:35 2014
@@ -30,7 +30,7 @@ public class TestICUTokenizerCJK extends
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new ICUTokenizer());
+ return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true)));
}
};
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java Sun May 4 12:19:35 2014
@@ -33,7 +33,7 @@ public class TestICUTokenizerFactory ext
Reader reader = new StringReader("à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ This is a test àºàº§à»àº²àºàºàº");
ICUTokenizerFactory factory = new ICUTokenizerFactory(new HashMap<String,String>());
factory.inform(new ClasspathResourceLoader(getClass()));
- Tokenizer stream = factory.create();
+ Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ",
@@ -48,7 +48,7 @@ public class TestICUTokenizerFactory ext
args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(this.getClass()));
- Tokenizer stream = factory.create();
+ Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"", "+2=3$5,&813", "!@#%$^)(*@#$" },
@@ -62,7 +62,7 @@ public class TestICUTokenizerFactory ext
args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(getClass()));
- Tokenizer stream = factory.create();
+ Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "One-two", "punch",
@@ -82,7 +82,7 @@ public class TestICUTokenizerFactory ext
args.put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(getClass()));
- Tokenizer stream = factory.create();
+ Tokenizer stream = factory.create(newAttributeFactory());
stream.setReader(reader);
assertTokenStreamContents(stream, new String[] { "Some", "English",
"Ðемного ÑÑÑÑкий. ",
Modified: lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java Sun May 4 12:19:35 2014
@@ -41,7 +41,7 @@ public class TestWithCJKBigramFilter ext
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
TokenStream result = new CJKBigramFilter(source);
return new TokenStreamComponents(source, new StopFilter(TEST_VERSION_CURRENT, result, CharArraySet.EMPTY_SET));
}
@@ -56,7 +56,7 @@ public class TestWithCJKBigramFilter ext
private Analyzer analyzer2 = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
// we put this before the CJKBigramFilter, because the normalization might combine
// some halfwidth katakana forms, which will affect the bigramming.
TokenStream result = new ICUNormalizer2Filter(source);
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java Sun May 4 12:19:35 2014
@@ -36,7 +36,7 @@ public class TestExtendedMode extends Ba
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, true, Mode.EXTENDED);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.EXTENDED);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java Sun May 4 12:19:35 2014
@@ -32,7 +32,7 @@ public class TestJapaneseBaseFormFilter
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.DEFAULT_MODE);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE);
return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer));
}
};
@@ -48,7 +48,7 @@ public class TestJapaneseBaseFormFilter
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new JapaneseTokenizer(null, true, JapaneseTokenizer.DEFAULT_MODE);
+ Tokenizer source = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new JapaneseBaseFormFilter(sink));
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java Sun May 4 12:19:35 2014
@@ -32,7 +32,7 @@ public class TestJapaneseBaseFormFilterF
public void testBasics() throws IOException {
JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap<String,String>());
tokenizerFactory.inform(new StringMockResourceLoader(""));
- TokenStream ts = tokenizerFactory.create();
+ TokenStream ts = tokenizerFactory.create(newAttributeFactory());
((Tokenizer)ts).setReader(new StringReader("ããã¯ã¾ã å®é¨æ®µéã«ããã¾ã"));
JapaneseBaseFormFilterFactory factory = new JapaneseBaseFormFilterFactory(new HashMap<String,String>());
ts = factory.create(ts);
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java Sun May 4 12:19:35 2014
@@ -45,7 +45,7 @@ public class TestJapaneseIterationMarkCh
private Analyzer japaneseAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, false, JapaneseTokenizer.Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java Sun May 4 12:19:35 2014
@@ -50,7 +50,7 @@ public class TestJapaneseIterationMarkCh
CharFilter filter = filterFactory.create(
new StringReader("æã
馬鹿ã
ã
ããã¨ããããããã¹ã¾")
);
- TokenStream tokenStream = tokenizerFactory.create();
+ TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"ææ", "馬鹿馬鹿ãã", "ã¨ããã©ãã", "ã", "ã¹ãº"});
}
@@ -67,7 +67,7 @@ public class TestJapaneseIterationMarkCh
CharFilter filter = filterFactory.create(
new StringReader("æã
馬鹿ã
ã
ããã¨ããããããã¹ã¾")
);
- TokenStream tokenStream = tokenizerFactory.create();
+ TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"ææ", "馬鹿馬鹿ãã", "ã¨ãã", "ã", "ã", "ã", "ãã¹", "ã¾"});
}
@@ -84,7 +84,7 @@ public class TestJapaneseIterationMarkCh
CharFilter filter = filterFactory.create(
new StringReader("æã
馬鹿ã
ã
ããã¨ããããããã¹ã¾")
);
- TokenStream tokenStream = tokenizerFactory.create();
+ TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"æã
", "馬鹿", "ã
", "ã
", "ãã", "ã¨ããã©ãã", "ã", "ã¹ãº"});
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java Sun May 4 12:19:35 2014
@@ -32,7 +32,7 @@ public class TestJapaneseKatakanaStemFil
public void testKatakanaStemming() throws IOException {
JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap<String,String>());
tokenizerFactory.inform(new StringMockResourceLoader(""));
- TokenStream tokenStream = tokenizerFactory.create();
+ TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
((Tokenizer)tokenStream).setReader(new StringReader("æå¾æ¥ãã¼ãã£ã¼ã«è¡ãäºå®ãããã峿¸é¤¨ã§è³æãã³ãã¼ãã¾ããã"));
JapaneseKatakanaStemFilterFactory filterFactory = new JapaneseKatakanaStemFilterFactory(new HashMap<String,String>());;
assertTokenStreamContents(filterFactory.create(tokenStream),
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java Sun May 4 12:19:35 2014
@@ -35,7 +35,7 @@ public class TestJapaneseReadingFormFilt
private Analyzer katakanaAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false));
}
};
@@ -43,7 +43,7 @@ public class TestJapaneseReadingFormFilt
private Analyzer romajiAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, true));
}
};
@@ -59,7 +59,7 @@ public class TestJapaneseReadingFormFilt
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
TokenStream stream = new CJKWidthFilter(tokenizer);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
}
@@ -79,7 +79,7 @@ public class TestJapaneseReadingFormFilt
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
TokenStream stream = new CJKWidthFilter(tokenizer);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java Sun May 4 12:19:35 2014
@@ -62,7 +62,7 @@ public class TestJapaneseTokenizer exten
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -70,7 +70,7 @@ public class TestJapaneseTokenizer exten
private Analyzer analyzerNormal = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.NORMAL);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.NORMAL);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -78,7 +78,7 @@ public class TestJapaneseTokenizer exten
private Analyzer analyzerNoPunct = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(readDict(), true, Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -86,7 +86,7 @@ public class TestJapaneseTokenizer exten
private Analyzer extendedModeAnalyzerNoPunct = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(readDict(), true, Mode.EXTENDED);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.EXTENDED);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -202,7 +202,7 @@ public class TestJapaneseTokenizer exten
new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
return new TokenStreamComponents(tokenizer, graph);
}
@@ -352,7 +352,7 @@ public class TestJapaneseTokenizer exten
final Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- JapaneseTokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
+ JapaneseTokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
tokenizer.setGraphvizFormatter(gv2);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java Sun May 4 12:19:35 2014
@@ -34,7 +34,7 @@ public class TestJapaneseTokenizerFactor
public void testSimple() throws IOException {
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap<String,String>());
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create();
+ TokenStream ts = factory.create(newAttributeFactory());
((Tokenizer)ts).setReader(new StringReader("ããã¯æ¬ã§ã¯ãªã"));
assertTokenStreamContents(ts,
new String[] { "ãã", "ã¯", "æ¬", "ã§", "ã¯", "ãªã" },
@@ -49,7 +49,7 @@ public class TestJapaneseTokenizerFactor
public void testDefaults() throws IOException {
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap<String,String>());
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create();
+ TokenStream ts = factory.create(newAttributeFactory());
((Tokenizer)ts).setReader(new StringReader("ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢"));
assertTokenStreamContents(ts,
new String[] { "ã·ãã¢", "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢", "ã½ããã¦ã§ã¢", "ã¨ã³ã¸ãã¢" }
@@ -64,7 +64,7 @@ public class TestJapaneseTokenizerFactor
args.put("mode", "normal");
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create();
+ TokenStream ts = factory.create(newAttributeFactory());
((Tokenizer)ts).setReader(new StringReader("ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢"));
assertTokenStreamContents(ts,
new String[] { "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢" }
@@ -85,7 +85,7 @@ public class TestJapaneseTokenizerFactor
args.put("userDictionary", "userdict.txt");
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
factory.inform(new StringMockResourceLoader(userDict));
- TokenStream ts = factory.create();
+ TokenStream ts = factory.create(newAttributeFactory());
((Tokenizer)ts).setReader(new StringReader("é¢è¥¿å½é空港ã«è¡ã£ã"));
assertTokenStreamContents(ts,
new String[] { "é¢è¥¿", "å½é", "空港", "ã«", "è¡ã£", "ã" }
@@ -100,7 +100,7 @@ public class TestJapaneseTokenizerFactor
args.put("discardPunctuation", "false");
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create();
+ TokenStream ts = factory.create(newAttributeFactory());
((Tokenizer)ts).setReader(new StringReader("ä»ãã«ã¦ã§ã¼ã«ãã¾ãããæ¥é±ã®é æ¥æ¬ã«æ»ãã¾ããæ¥½ãã¿ã«ãã¦ãã¾ãï¼ã寿å¸ãé£ã¹ãããªããã"));
assertTokenStreamContents(ts,
new String[] { "ä»", "ãã«ã¦ã§ã¼", "ã«", "ã", "ã¾ã", "ã", "ã",
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java Sun May 4 12:19:35 2014
@@ -34,7 +34,7 @@ public class TestSearchMode extends Base
private final Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(null, true, Mode.SEARCH);
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java Sun May 4 12:19:35 2014
@@ -17,7 +17,6 @@
package org.apache.lucene.analysis.phonetic;
import java.io.IOException;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -25,54 +24,47 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.util.TestUtil;
public class DoubleMetaphoneFilterTest extends BaseTokenStreamTestCase {
-
- private TokenStream whitespaceTokenizer(String data) throws IOException {
- WhitespaceTokenizer whitespaceTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
- whitespaceTokenizer.setReader(new StringReader(data));
- return whitespaceTokenizer;
- }
public void testSize4FalseInject() throws Exception {
- TokenStream stream = whitespaceTokenizer("international");
+ TokenStream stream = whitespaceMockTokenizer("international");
TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);
assertTokenStreamContents(filter, new String[] { "ANTR" });
}
public void testSize4TrueInject() throws Exception {
- TokenStream stream = whitespaceTokenizer("international");
+ TokenStream stream = whitespaceMockTokenizer("international");
TokenStream filter = new DoubleMetaphoneFilter(stream, 4, true);
assertTokenStreamContents(filter, new String[] { "international", "ANTR" });
}
public void testAlternateInjectFalse() throws Exception {
- TokenStream stream = whitespaceTokenizer("Kuczewski");
+ TokenStream stream = whitespaceMockTokenizer("Kuczewski");
TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);
assertTokenStreamContents(filter, new String[] { "KSSK", "KXFS" });
}
public void testSize8FalseInject() throws Exception {
- TokenStream stream = whitespaceTokenizer("international");
+ TokenStream stream = whitespaceMockTokenizer("international");
TokenStream filter = new DoubleMetaphoneFilter(stream, 8, false);
assertTokenStreamContents(filter, new String[] { "ANTRNXNL" });
}
public void testNonConvertableStringsWithInject() throws Exception {
- TokenStream stream = whitespaceTokenizer("12345 #$%@#^%&");
+ TokenStream stream = whitespaceMockTokenizer("12345 #$%@#^%&");
TokenStream filter = new DoubleMetaphoneFilter(stream, 8, true);
assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&" });
}
public void testNonConvertableStringsWithoutInject() throws Exception {
- TokenStream stream = whitespaceTokenizer("12345 #$%@#^%&");
+ TokenStream stream = whitespaceMockTokenizer("12345 #$%@#^%&");
TokenStream filter = new DoubleMetaphoneFilter(stream, 8, false);
assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&" });
// should have something after the stream
- stream = whitespaceTokenizer("12345 #$%@#^%& hello");
+ stream = whitespaceMockTokenizer("12345 #$%@#^%& hello");
filter = new DoubleMetaphoneFilter(stream, 8, false);
assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&", "HL" });
}
Modified: lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java Sun May 4 12:19:35 2014
@@ -110,7 +110,7 @@ public class TestBeiderMorseFilter exten
}
public void testCustomAttribute() throws IOException {
- TokenStream stream = new KeywordTokenizer();
+ TokenStream stream = new MockTokenizer(MockTokenizer.KEYWORD, false);
((Tokenizer)stream).setReader(new StringReader("D'Angelo"));
stream = new PatternKeywordMarkerFilter(stream, Pattern.compile(".*"));
stream = new BeiderMorseFilter(stream, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true));
Modified: lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestHMMChineseTokenizerFactory.java Sun May 4 12:19:35 2014
@@ -34,7 +34,7 @@ public class TestHMMChineseTokenizerFact
public void testSimple() throws Exception {
Reader reader = new StringReader("æè´ä¹°äºéå
·åæè£
ã");
TokenizerFactory factory = new HMMChineseTokenizerFactory(new HashMap<String,String>());
- Tokenizer tokenizer = factory.create();
+ Tokenizer tokenizer = factory.create(newAttributeFactory());
tokenizer.setReader(reader);
// TODO: fix smart chinese to not emit punctuation tokens
// at the moment: you have to clean up with WDF, or use the stoplist, etc
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestToken.java Sun May 4 12:19:35 2014
@@ -40,16 +40,9 @@ public class TestToken extends LuceneTes
assertEquals("word", t.type());
assertEquals(0, t.getFlags());
- t = new Token(6, 22);
- t.copyBuffer(content, 0, content.length);
- assertEquals("hello", t.toString());
- assertEquals("hello", t.toString());
- assertEquals(6, t.startOffset());
- assertEquals(22, t.endOffset());
- assertEquals("word", t.type());
- assertEquals(0, t.getFlags());
-
- t = new Token(6, 22, 7);
+ t = new Token();
+ t.setOffset(6, 22);
+ t.setFlags(7);
t.copyBuffer(content, 0, content.length);
assertEquals("hello", t.toString());
assertEquals("hello", t.toString());
@@ -58,7 +51,9 @@ public class TestToken extends LuceneTes
assertEquals("word", t.type());
assertEquals(7, t.getFlags());
- t = new Token(6, 22, "junk");
+ t = new Token();
+ t.setOffset(6, 22);
+ t.setType("junk");
t.copyBuffer(content, 0, content.length);
assertEquals("hello", t.toString());
assertEquals("hello", t.toString());
@@ -174,7 +169,8 @@ public class TestToken extends LuceneTes
}
public void testClone() throws Exception {
- Token t = new Token(0, 5);
+ Token t = new Token();
+ t.setOffset(0, 5);
char[] content = "hello".toCharArray();
t.copyBuffer(content, 0, 5);
char[] buf = t.buffer();
@@ -195,7 +191,8 @@ public class TestToken extends LuceneTes
assertEquals("", t.toString());
assertEquals("", copy.toString());
- t = new Token(0, 5);
+ t = new Token();
+ t.setOffset(0, 5);
char[] content = "hello".toCharArray();
t.copyBuffer(content, 0, 5);
char[] buf = t.buffer();
@@ -245,7 +242,8 @@ public class TestToken extends LuceneTes
}
public void testAttributeReflection() throws Exception {
- Token t = new Token("foobar", 6, 22, 8);
+ Token t = new Token("foobar", 6, 22);
+ t.setFlags(8);
TestUtil.assertAttributeReflection(t,
new HashMap<String, Object>() {{
put(CharTermAttribute.class.getName() + "#term", "foobar");
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java Sun May 4 12:19:35 2014
@@ -591,7 +591,6 @@ public class TestPayloads extends Lucene
Field field = new TextField("field", "", Field.Store.NO);
TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("here we go"));
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
writer.addDocument(doc);
@@ -603,7 +602,6 @@ public class TestPayloads extends Lucene
writer.addDocument(doc);
ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("another"));
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
writer.addDocument(doc);
DirectoryReader reader = writer.getReader();
@@ -625,7 +623,6 @@ public class TestPayloads extends Lucene
Field field = new TextField("field", "", Field.Store.NO);
TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("here we go"));
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
Field field2 = new TextField("field", "", Field.Store.NO);
@@ -638,8 +635,6 @@ public class TestPayloads extends Lucene
Field field3 = new TextField("field", "", Field.Store.NO);
ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("nopayload"));
-
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field3.setTokenStream(ts);
doc.add(field3);
writer.addDocument(doc);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java Sun May 4 12:19:35 2014
@@ -51,7 +51,6 @@ public class TestPayloadsOnVectors exten
Field field = new Field("field", "", customType);
TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("here we go"));
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
writer.addDocument(doc);
@@ -65,7 +64,6 @@ public class TestPayloadsOnVectors exten
ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("another"));
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
writer.addDocument(doc);
@@ -96,7 +94,6 @@ public class TestPayloadsOnVectors exten
Field field = new Field("field", "", customType);
TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("here we go"));
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
Field field2 = new Field("field", "", customType);
@@ -109,7 +106,6 @@ public class TestPayloadsOnVectors exten
Field field3 = new Field("field", "", customType);
ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
((Tokenizer)ts).setReader(new StringReader("nopayload"));
- assertFalse(ts.hasAttribute(PayloadAttribute.class));
field3.setTokenStream(ts);
doc.add(field3);
writer.addDocument(doc);
Modified: lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java Sun May 4 12:19:35 2014
@@ -67,7 +67,8 @@ public class TokenGroup {
tot += score;
}
}
- Token token = new Token(termStartOffset, termEndOffset);
+ Token token = new Token();
+ token.setOffset(termStartOffset, termEndOffset);
token.setEmpty().append(termAtt);
tokens[numTokens] = token;
scores[numTokens] = score;
Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Sun May 4 12:19:35 2014
@@ -314,10 +314,10 @@ public class HighlighterPhraseTest exten
public void reset() {
this.i = -1;
this.tokens = new Token[] {
- new Token(new char[] { 't', 'h', 'e' }, 0, 3, 0, 3),
- new Token(new char[] { 'f', 'o', 'x' }, 0, 3, 4, 7),
- new Token(new char[] { 'd', 'i', 'd' }, 0, 3, 8, 11),
- new Token(new char[] { 'j', 'u', 'm', 'p' }, 0, 4, 16, 20) };
+ new Token("the", 0, 3),
+ new Token("fox", 4, 7),
+ new Token("did", 8, 11),
+ new Token("jump", 16, 20) };
this.tokens[3].setPositionIncrement(2);
}
}
@@ -354,10 +354,10 @@ public class HighlighterPhraseTest exten
public void reset() {
this.i = -1;
this.tokens = new Token[] {
- new Token(new char[] { 't', 'h', 'e' }, 0, 3, 0, 3),
- new Token(new char[] { 'f', 'o', 'x' }, 0, 3, 4, 7),
- new Token(new char[] { 'j', 'u', 'm', 'p' }, 0, 4, 8, 14),
- new Token(new char[] { 'j', 'u', 'm', 'p', 'e', 'd' }, 0, 6, 8, 14) };
+ new Token("the", 0, 3),
+ new Token("fox", 4, 7),
+ new Token("jump", 8, 14),
+ new Token("jumped", 8, 14) };
this.tokens[3].setPositionIncrement(0);
}
}
Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Sun May 4 12:19:35 2014
@@ -2013,7 +2013,8 @@ final class SynonymTokenizer extends Tok
}
st = new StringTokenizer(expansions, ",");
if (st.hasMoreTokens()) {
- currentRealToken = new Token(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
+ currentRealToken = new Token();
+ currentRealToken.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
currentRealToken.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
}
Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java Sun May 4 12:19:35 2014
@@ -78,12 +78,12 @@ public class TokenSourcesTest extends Lu
public void reset() {
this.i = -1;
this.tokens = new Token[] {
- new Token(new char[] {'t', 'h', 'e'}, 0, 3, 0, 3),
- new Token(new char[] {'{', 'f', 'o', 'x', '}'}, 0, 5, 0, 7),
- new Token(new char[] {'f', 'o', 'x'}, 0, 3, 4, 7),
- new Token(new char[] {'d', 'i', 'd'}, 0, 3, 8, 11),
- new Token(new char[] {'n', 'o', 't'}, 0, 3, 12, 15),
- new Token(new char[] {'j', 'u', 'm', 'p'}, 0, 4, 16, 20)};
+ new Token("the", 0, 3),
+ new Token("{fox}", 0, 7),
+ new Token("fox", 4, 7),
+ new Token("did", 8, 11),
+ new Token("not", 12, 15),
+ new Token("jump", 16, 20)};
this.tokens[1].setPositionIncrement(0);
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Sun May 4 12:19:35 2014
@@ -39,6 +39,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -933,5 +934,18 @@ public abstract class BaseTokenStreamTes
mockTokenizer.setReader(new StringReader(input));
return mockTokenizer;
}
-
+
+ /** Returns a new AttributeFactory impl */
+ public static AttributeFactory newAttributeFactory(Random random) {
+ if (random.nextBoolean()) {
+ return Token.TOKEN_ATTRIBUTE_FACTORY;
+ } else {
+ return AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
+ }
+ }
+
+ /** Returns a new AttributeFactory impl */
+ public static AttributeFactory newAttributeFactory() {
+ return newAttributeFactory(random());
+ }
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java?rev=1592339&r1=1592338&r2=1592339&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java Sun May 4 12:19:35 2014
@@ -98,7 +98,7 @@ public class MockTokenizer extends Token
}
public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, runAutomaton, lowerCase, maxTokenLength);
+ this(BaseTokenStreamTestCase.newAttributeFactory(), runAutomaton, lowerCase, maxTokenLength);
}
public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {