You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/01/09 14:27:36 UTC
svn commit: r1556801 [6/10] - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/
lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/
lucene/analysis/common/src/java/org/apache/lucene/analysis/br/ luc...
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java Thu Jan 9 13:27:29 2014
@@ -31,7 +31,8 @@ public class TestPathHierarchyTokenizer
public void testBasic() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a", "/a/b", "/a/b/c"},
new int[]{0, 0, 0},
@@ -42,7 +43,8 @@ public class TestPathHierarchyTokenizer
public void testEndOfDelimiter() throws Exception {
String path = "/a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"/a", "/a/b", "/a/b/c", "/a/b/c/"},
new int[]{0, 0, 0, 0},
@@ -53,7 +55,8 @@ public class TestPathHierarchyTokenizer
public void testStartOfChar() throws Exception {
String path = "a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"a", "a/b", "a/b/c"},
new int[]{0, 0, 0},
@@ -64,7 +67,8 @@ public class TestPathHierarchyTokenizer
public void testStartOfCharEndOfDelimiter() throws Exception {
String path = "a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"a", "a/b", "a/b/c", "a/b/c/"},
new int[]{0, 0, 0, 0},
@@ -75,7 +79,8 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimiter() throws Exception {
String path = "/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{0},
@@ -86,7 +91,8 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimiters() throws Exception {
String path = "//";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/", "//"},
new int[]{0, 0},
@@ -97,7 +103,8 @@ public class TestPathHierarchyTokenizer
public void testReplace() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), '/', '\\' );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( '/', '\\' );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"\\a", "\\a\\b", "\\a\\b\\c"},
new int[]{0, 0, 0},
@@ -108,7 +115,8 @@ public class TestPathHierarchyTokenizer
public void testWindowsPath() throws Exception {
String path = "c:\\a\\b\\c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), '\\', '\\' );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( '\\', '\\' );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"c:", "c:\\a", "c:\\a\\b", "c:\\a\\b\\c"},
new int[]{0, 0, 0, 0},
@@ -123,7 +131,8 @@ public class TestPathHierarchyTokenizer
NormalizeCharMap normMap = builder.build();
String path = "c:\\a\\b\\c";
Reader cs = new MappingCharFilter(normMap, new StringReader(path));
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader(cs);
assertTokenStreamContents(t,
new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
new int[]{0, 0, 0, 0},
@@ -134,7 +143,8 @@ public class TestPathHierarchyTokenizer
public void testBasicSkip() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c"},
new int[]{2, 2},
@@ -145,7 +155,8 @@ public class TestPathHierarchyTokenizer
public void testEndOfDelimiterSkip() throws Exception {
String path = "/a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c", "/b/c/"},
new int[]{2, 2, 2},
@@ -156,7 +167,8 @@ public class TestPathHierarchyTokenizer
public void testStartOfCharSkip() throws Exception {
String path = "a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c"},
new int[]{1, 1},
@@ -167,7 +179,8 @@ public class TestPathHierarchyTokenizer
public void testStartOfCharEndOfDelimiterSkip() throws Exception {
String path = "a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c", "/b/c/"},
new int[]{1, 1, 1},
@@ -178,7 +191,8 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimiterSkip() throws Exception {
String path = "/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{},
new int[]{},
@@ -189,7 +203,8 @@ public class TestPathHierarchyTokenizer
public void testOnlyDelimitersSkip() throws Exception {
String path = "//";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader( new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{1},
@@ -202,8 +217,8 @@ public class TestPathHierarchyTokenizer
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -215,8 +230,8 @@ public class TestPathHierarchyTokenizer
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,8 @@ public class TestReversePathHierarchyTok
public void testBasicReverse() throws Exception {
String path = "/a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/b/c", "a/b/c", "b/c", "c"},
new int[]{0, 1, 3, 5},
@@ -41,7 +42,8 @@ public class TestReversePathHierarchyTok
public void testEndOfDelimiterReverse() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/b/c/", "a/b/c/", "b/c/", "c/"},
new int[]{0, 1, 3, 5},
@@ -52,7 +54,8 @@ public class TestReversePathHierarchyTok
public void testStartOfCharReverse() throws Exception {
String path = "a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/c", "b/c", "c"},
new int[]{0, 2, 4},
@@ -63,7 +66,8 @@ public class TestReversePathHierarchyTok
public void testStartOfCharEndOfDelimiterReverse() throws Exception {
String path = "a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/c/", "b/c/", "c/"},
new int[]{0, 2, 4},
@@ -74,7 +78,8 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimiterReverse() throws Exception {
String path = "/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{0},
@@ -85,7 +90,8 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimitersReverse() throws Exception {
String path = "//";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"//", "/"},
new int[]{0, 1},
@@ -96,7 +102,9 @@ public class TestReversePathHierarchyTok
public void testEndOfDelimiterReverseSkip() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
+ new StringReader(path);
assertTokenStreamContents(t,
new String[]{"/a/b/", "a/b/", "b/"},
new int[]{0, 1, 3},
@@ -107,7 +115,8 @@ public class TestReversePathHierarchyTok
public void testStartOfCharReverseSkip() throws Exception {
String path = "a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/", "b/"},
new int[]{0, 2},
@@ -118,7 +127,8 @@ public class TestReversePathHierarchyTok
public void testStartOfCharEndOfDelimiterReverseSkip() throws Exception {
String path = "a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/", "b/"},
new int[]{0, 2},
@@ -129,7 +139,8 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimiterReverseSkip() throws Exception {
String path = "/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{},
new int[]{},
@@ -140,7 +151,8 @@ public class TestReversePathHierarchyTok
public void testOnlyDelimitersReverseSkip() throws Exception {
String path = "//";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{0},
@@ -151,7 +163,8 @@ public class TestReversePathHierarchyTok
public void testReverseSkip2() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 2 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 2 );
+ t.setReader( new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/", "a/"},
new int[]{0, 1},
@@ -164,8 +177,8 @@ public class TestReversePathHierarchyTok
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new ReversePathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -177,8 +190,8 @@ public class TestReversePathHierarchyTok
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new ReversePathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java Thu Jan 9 13:27:29 2014
@@ -597,10 +597,8 @@ public class TestPatternCaptureGroupToke
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader,
- MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
new PatternCaptureGroupTokenFilter(tokenizer, false,
Pattern.compile("((..)(..))")));
@@ -617,9 +615,10 @@ public class TestPatternCaptureGroupToke
for (int i = 0; i < regexes.length; i++) {
patterns[i] = Pattern.compile(regexes[i]);
}
- TokenStream ts = new PatternCaptureGroupTokenFilter(new MockTokenizer(
- new StringReader(input), MockTokenizer.WHITESPACE, false),
- preserveOriginal, patterns);
+
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader( new StringReader(input));
+ TokenStream ts = new PatternCaptureGroupTokenFilter(tokenizer, preserveOriginal, patterns);
assertTokenStreamContents(ts, tokens, startOffsets, endOffsets, positions);
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java Thu Jan 9 13:27:29 2014
@@ -139,7 +139,7 @@ public class TestPatternReplaceCharFilte
final String BLOCK = "this is test.";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1$2$3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "test." },
new int[] { 0, 5, 8 },
@@ -153,7 +153,7 @@ public class TestPatternReplaceCharFilte
final String BLOCK = "aa bb cc";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[] {});
}
@@ -164,7 +164,7 @@ public class TestPatternReplaceCharFilte
final String BLOCK = "aa bb cc";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2#$3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa#bb#cc" },
new int[] { 0 },
@@ -180,7 +180,7 @@ public class TestPatternReplaceCharFilte
final String BLOCK = "aa bb cc dd";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1##$2###$3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa##bb###cc", "dd" },
new int[] { 0, 9 },
@@ -195,7 +195,7 @@ public class TestPatternReplaceCharFilte
final String BLOCK = " a a";
CharFilter cs = new PatternReplaceCharFilter( pattern("a"), "aa",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa", "aa" },
new int[] { 1, 4 },
@@ -211,7 +211,7 @@ public class TestPatternReplaceCharFilte
final String BLOCK = "aa bb cc dd";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa#bb", "dd" },
new int[] { 0, 12 },
@@ -227,7 +227,7 @@ public class TestPatternReplaceCharFilte
final String BLOCK = " aa bb cc --- aa bb aa bb cc";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1 $2 $3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa", "bb", "cc", "---", "aa", "bb", "aa", "bb", "cc" },
new int[] { 2, 6, 9, 11, 15, 18, 21, 25, 29 },
@@ -247,7 +247,7 @@ public class TestPatternReplaceCharFilte
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)"), "$1##$2",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa##bb", "cc", "---", "aa##bb", "aa.", "bb", "aa##bb", "cc" },
new int[] { 2, 8, 11, 15, 21, 25, 28, 36 },
@@ -265,7 +265,7 @@ public class TestPatternReplaceCharFilte
new StringReader( BLOCK ) );
cs = new PatternReplaceCharFilter( pattern("bb"), "b", cs );
cs = new PatternReplaceCharFilter( pattern("ccc"), "c", cs );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa", "b", "-", "c", ".", "---", "b", "aa", ".", "c", "c", "b" },
new int[] { 1, 3, 6, 8, 12, 14, 18, 21, 23, 25, 29, 33 },
@@ -307,8 +307,8 @@ public class TestPatternReplaceCharFilte
final String replacement = _TestUtil.randomSimpleString(random);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -37,7 +37,7 @@ public class TestPatternReplaceCharFilte
reader = charFilterFactory("PatternReplace",
"pattern", "(aa)\\s+(bb)\\s+(cc)",
"replacement", "$1$2$3").create(reader);
- TokenStream ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(reader);
assertTokenStreamContents(ts,
new String[] { "this", "is", "test." },
new int[] { 0, 5, 8 },
@@ -50,7 +50,7 @@ public class TestPatternReplaceCharFilte
Reader reader = new StringReader("aa bb cc");
reader = charFilterFactory("PatternReplace",
"pattern", "(aa)\\s+(bb)\\s+(cc)").create(reader);
- TokenStream ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(reader);
assertTokenStreamContents(ts, new String[] {});
}
@@ -62,7 +62,7 @@ public class TestPatternReplaceCharFilte
reader = charFilterFactory("PatternReplace",
"pattern", "(aa)\\s+(bb)\\s+(cc)",
"replacement", "$1#$2#$3").create(reader);
- TokenStream ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(reader);
assertTokenStreamContents(ts,
new String[] { "aa#bb#cc" },
new int[] { 0 },
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java Thu Jan 9 13:27:29 2014
@@ -25,18 +25,16 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.core.KeywordTokenizer;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.regex.Pattern;
/**
*/
public class TestPatternReplaceFilter extends BaseTokenStreamTestCase {
-
+
public void testReplaceAll() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
"-", true);
assertTokenStreamContents(ts,
@@ -46,7 +44,7 @@ public class TestPatternReplaceFilter ex
public void testReplaceFirst() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
"-", false);
assertTokenStreamContents(ts,
@@ -56,7 +54,7 @@ public class TestPatternReplaceFilter ex
public void testStripFirst() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
null, false);
assertTokenStreamContents(ts,
@@ -66,7 +64,7 @@ public class TestPatternReplaceFilter ex
public void testStripAll() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
null, true);
assertTokenStreamContents(ts,
@@ -76,7 +74,7 @@ public class TestPatternReplaceFilter ex
public void testReplaceAllWithBackRef() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("(a*)b"),
"$1\\$", true);
assertTokenStreamContents(ts,
@@ -87,8 +85,8 @@ public class TestPatternReplaceFilter ex
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false);
return new TokenStreamComponents(tokenizer, filter);
}
@@ -97,8 +95,8 @@ public class TestPatternReplaceFilter ex
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true);
return new TokenStreamComponents(tokenizer, filter);
}
@@ -109,8 +107,8 @@ public class TestPatternReplaceFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -31,7 +31,7 @@ public class TestPatternReplaceFilterFac
public void testReplaceAll() throws Exception {
Reader reader = new StringReader("aabfooaabfooabfoob ab caaaaaaaaab");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PatternReplace",
"pattern", "a*b",
"replacement", "-").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java Thu Jan 9 13:27:29 2014
@@ -53,7 +53,8 @@ public class TestPatternTokenizer extend
};
for( String[] test : tests ) {
- TokenStream stream = new PatternTokenizer(new StringReader(test[2]), Pattern.compile(test[1]), Integer.parseInt(test[0]));
+ TokenStream stream = new PatternTokenizer(Pattern.compile(test[1]), Integer.parseInt(test[0]));
+ ((Tokenizer)stream).setReader(new StringReader(test[2]));
String out = tsToString( stream );
// System.out.println( test[2] + " ==> " + out );
@@ -85,7 +86,8 @@ public class TestPatternTokenizer extend
CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
// create PatternTokenizer
- TokenStream stream = new PatternTokenizer(charStream, Pattern.compile("[,;/\\s]+"), -1);
+ Tokenizer stream = new PatternTokenizer(Pattern.compile("[,;/\\s]+"), -1);
+ stream.setReader(charStream);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther", "is", "here" },
new int[] { 0, 13, 26, 29 },
@@ -93,7 +95,8 @@ public class TestPatternTokenizer extend
INPUT.length());
charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
- stream = new PatternTokenizer(charStream, Pattern.compile("Günther"), 0);
+ stream = new PatternTokenizer(Pattern.compile("Günther"), 0);
+ stream.setReader(charStream);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther" },
new int[] { 0, 13 },
@@ -128,8 +131,8 @@ public class TestPatternTokenizer extend
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), -1);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PatternTokenizer(Pattern.compile("a"), -1);
return new TokenStreamComponents(tokenizer);
}
};
@@ -137,8 +140,8 @@ public class TestPatternTokenizer extend
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), 0);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PatternTokenizer(Pattern.compile("a"), 0);
return new TokenStreamComponents(tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java Thu Jan 9 13:27:29 2014
@@ -20,7 +20,7 @@ package org.apache.lucene.analysis.patte
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/** Simple Tests to ensure this factory is working */
@@ -28,7 +28,8 @@ public class TestPatternTokenizerFactory
public void testFactory() throws Exception {
final Reader reader = new StringReader("Günther Günther is here");
// create PatternTokenizer
- TokenStream stream = tokenizerFactory("Pattern", "pattern", "[,;/\\s]+").create(reader);
+ Tokenizer stream = tokenizerFactory("Pattern", "pattern", "[,;/\\s]+").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther", "is", "here" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java Thu Jan 9 13:27:29 2014
@@ -16,6 +16,7 @@ package org.apache.lucene.analysis.paylo
* limitations under the License.
*/
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -25,12 +26,12 @@ import org.apache.lucene.util.LuceneTest
import java.io.StringReader;
-public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {
+public class DelimitedPayloadTokenFilterTest extends BaseTokenStreamTestCase {
public void testPayloads() throws Exception {
String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
- (new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(test),
DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
@@ -54,7 +55,7 @@ public class DelimitedPayloadTokenFilter
String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
- (new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(test),
DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
filter.reset();
assertTermEquals("The", filter, null);
@@ -75,7 +76,7 @@ public class DelimitedPayloadTokenFilter
public void testFloatEncoding() throws Exception {
String test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
- DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new FloatEncoder());
+ DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(whitespaceMockTokenizer(test), '|', new FloatEncoder());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
filter.reset();
@@ -96,7 +97,7 @@ public class DelimitedPayloadTokenFilter
public void testIntEncoding() throws Exception {
String test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
- DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder());
+ DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(whitespaceMockTokenizer(test), '|', new IntegerEncoder());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
filter.reset();
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java Thu Jan 9 13:27:29 2014
@@ -32,7 +32,9 @@ public class NumericPayloadTokenFilterTe
public void test() throws IOException {
String test = "The quick red fox jumped over the lazy brown dogs";
- NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)), 3, "D");
+ final MockTokenizer input = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ input.setReader(new StringReader(test));
+ NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(input), 3, "D");
boolean seenDogs = false;
CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TestDelimitedPayloadTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TestDelimitedPayloadTokenFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TestDelimitedPayloadTokenFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TestDelimitedPayloadTokenFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
@@ -30,7 +31,8 @@ public class TestDelimitedPayloadTokenFi
public void testEncoder() throws Exception {
Reader reader = new StringReader("the|0.1 quick|0.1 red|0.1");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);
stream.reset();
@@ -48,7 +50,8 @@ public class TestDelimitedPayloadTokenFi
public void testDelim() throws Exception {
Reader reader = new StringReader("the*0.1 quick*0.1 red*0.1");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("DelimitedPayload",
"encoder", "float",
"delimiter", "*").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java Thu Jan 9 13:27:29 2014
@@ -23,14 +23,13 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
-import java.io.StringReader;
public class TokenOffsetPayloadTokenFilterTest extends BaseTokenStreamTestCase {
public void test() throws IOException {
String test = "The quick red fox jumped over the lazy brown dogs";
- TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(whitespaceMockTokenizer(test));
int count = 0;
PayloadAttribute payloadAtt = nptf.getAttribute(PayloadAttribute.class);
OffsetAttribute offsetAtt = nptf.getAttribute(OffsetAttribute.class);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java Thu Jan 9 13:27:29 2014
@@ -32,7 +32,7 @@ public class TypeAsPayloadTokenFilterTes
public void test() throws IOException {
String test = "The quick red fox jumped over the lazy brown dogs";
- TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
+ TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(whitespaceMockTokenizer(test)));
int count = 0;
CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -40,9 +40,8 @@ import static org.apache.lucene.analysis
public class TestPortugueseLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new PortugueseLightStemFilter(result));
}
@@ -101,8 +100,8 @@ public class TestPortugueseLightStemFilt
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("quilométricas"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseLightStemFilter(sink));
}
@@ -118,8 +117,8 @@ public class TestPortugueseLightStemFilt
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PortugueseLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@ import org.apache.lucene.analysis.util.B
public class TestPortugueseLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("evidentemente");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("PortugueseLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "evident" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java Thu Jan 9 13:27:29 2014
@@ -40,9 +40,8 @@ import static org.apache.lucene.analysis
public class TestPortugueseMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(result));
}
@@ -75,8 +74,8 @@ public class TestPortugueseMinimalStemFi
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("quilométricas"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(sink));
}
@@ -92,8 +91,8 @@ public class TestPortugueseMinimalStemFi
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PortugueseMinimalStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@ import org.apache.lucene.analysis.util.B
public class TestPortugueseMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("questões");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("PortugueseMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "questão" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java Thu Jan 9 13:27:29 2014
@@ -39,9 +39,8 @@ import org.apache.lucene.analysis.util.C
public class TestPortugueseStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new PortugueseStemFilter(result));
}
@@ -74,8 +73,8 @@ public class TestPortugueseStemFilter ex
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("quilométricas"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseStemFilter(sink));
}
@@ -91,8 +90,8 @@ public class TestPortugueseStemFilter ex
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PortugueseStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestPortugueseStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("maluquice");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PortugueseStem").create(stream);
assertTokenStreamContents(stream, new String[] { "maluc" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java Thu Jan 9 13:27:29 2014
@@ -31,15 +31,15 @@ import org.apache.lucene.util.Version;
public class TestReverseStringFilter extends BaseTokenStreamTestCase {
public void testFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader("Do have a nice day"),
- MockTokenizer.WHITESPACE, false); // 1-4 length string
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false); // 1-4 length string
+ ((Tokenizer)stream).setReader(new StringReader("Do have a nice day"));
ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream);
assertTokenStreamContents(filter, new String[] { "oD", "evah", "a", "ecin", "yad" });
}
public void testFilterWithMark() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader("Do have a nice day"),
- MockTokenizer.WHITESPACE, false); // 1-4 length string
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false); // 1-4 length string
+ ((Tokenizer)stream).setReader(new StringReader("Do have a nice day"));
ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream, '\u0001');
assertTokenStreamContents(filter,
new String[] { "\u0001oD", "\u0001evah", "\u0001a", "\u0001ecin", "\u0001yad" });
@@ -97,8 +97,8 @@ public class TestReverseStringFilter ext
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
@@ -108,8 +108,8 @@ public class TestReverseStringFilter ext
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -33,7 +33,7 @@ public class TestReverseStringFilterFact
*/
public void testReversing() throws Exception {
Reader reader = new StringReader("simple test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ReverseString").create(stream);
assertTokenStreamContents(stream, new String[] { "elpmis", "tset" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java Thu Jan 9 13:27:29 2014
@@ -37,9 +37,8 @@ import static org.apache.lucene.analysis
public class TestRussianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new RussianLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@ public class TestRussianLightStemFilter
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("ÑнеÑгии"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new RussianLightStemFilter(sink));
}
@@ -70,8 +69,8 @@ public class TestRussianLightStemFilter
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new RussianLightStemFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.util.B
public class TestRussianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("жÑÑналÑ");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("RussianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "жÑÑнал" });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java Thu Jan 9 13:27:29 2014
@@ -981,7 +981,8 @@ public class ShingleFilterTest extends B
}
public void testReset() throws Exception {
- Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("please divide this sentence"));
+ Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ wsTokenizer.setReader(new StringReader("please divide this sentence"));
TokenStream filter = new ShingleFilter(wsTokenizer, 2);
assertTokenStreamContents(filter,
new String[]{"please","please divide","divide","divide this","this","this sentence","sentence"},
@@ -1105,8 +1106,8 @@ public class ShingleFilterTest extends B
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
}
};
@@ -1118,8 +1119,8 @@ public class ShingleFilterTest extends B
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
}
};
@@ -1129,8 +1130,8 @@ public class ShingleFilterTest extends B
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -33,7 +33,7 @@ public class TestShingleFilterFactory ex
*/
public void testDefaults() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle").create(stream);
assertTokenStreamContents(stream,
new String[] { "this", "this is", "is", "is a", "a", "a test", "test" }
@@ -45,7 +45,7 @@ public class TestShingleFilterFactory ex
*/
public void testNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"outputUnigrams", "false").create(stream);
assertTokenStreamContents(stream,
@@ -57,7 +57,7 @@ public class TestShingleFilterFactory ex
*/
public void testMaxShingleSize() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"maxShingleSize", "3").create(stream);
assertTokenStreamContents(stream,
@@ -71,7 +71,7 @@ public class TestShingleFilterFactory ex
*/
public void testMinShingleSize() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4").create(stream);
@@ -86,7 +86,7 @@ public class TestShingleFilterFactory ex
*/
public void testMinShingleSizeNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4",
@@ -100,7 +100,7 @@ public class TestShingleFilterFactory ex
*/
public void testEqualMinAndMaxShingleSize() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "3").create(stream);
@@ -113,7 +113,7 @@ public class TestShingleFilterFactory ex
*/
public void testEqualMinAndMaxShingleSizeNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "3",
@@ -127,7 +127,7 @@ public class TestShingleFilterFactory ex
*/
public void testTokenSeparator() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"tokenSeparator", "=BLAH=").create(stream);
assertTokenStreamContents(stream,
@@ -141,7 +141,7 @@ public class TestShingleFilterFactory ex
*/
public void testTokenSeparatorNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"tokenSeparator", "=BLAH=",
"outputUnigrams", "false").create(stream);
@@ -154,7 +154,7 @@ public class TestShingleFilterFactory ex
*/
public void testEmptyTokenSeparator() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"tokenSeparator", "").create(stream);
assertTokenStreamContents(stream,
@@ -167,7 +167,7 @@ public class TestShingleFilterFactory ex
*/
public void testMinShingleSizeAndTokenSeparator() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4",
@@ -186,7 +186,7 @@ public class TestShingleFilterFactory ex
*/
public void testMinShingleSizeAndTokenSeparatorNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4",
@@ -208,7 +208,7 @@ public class TestShingleFilterFactory ex
*/
public void testOutputUnigramsIfNoShingles() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"outputUnigrams", "false",
"outputUnigramsIfNoShingles", "true").create(stream);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizerTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizerTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizerTest.java Thu Jan 9 13:27:29 2014
@@ -29,7 +29,9 @@ public class DateRecognizerSinkTokenizer
public void test() throws IOException {
DateRecognizerSinkFilter sinkFilter = new DateRecognizerSinkFilter(new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT));
String test = "The quick red fox jumped over the lazy brown dogs on 7/11/2006 The dogs finally reacted on 7/12/2006";
- TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ final MockTokenizer input = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ input.setReader(new StringReader(test));
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(input);
TeeSinkTokenFilter.SinkTokenStream sink = tee.newSinkTokenStream(sinkFilter);
int count = 0;
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.util.Locale;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -126,7 +127,7 @@ public class TestTeeSinkTokenFilter exte
}
public void testGeneral() throws IOException {
- final TeeSinkTokenFilter source = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ final TeeSinkTokenFilter source = new TeeSinkTokenFilter(whitespaceMockTokenizer(buffer1.toString()));
final TokenStream sink1 = source.newSinkTokenStream();
final TokenStream sink2 = source.newSinkTokenStream(theFilter);
@@ -140,7 +141,7 @@ public class TestTeeSinkTokenFilter exte
}
public void testMultipleSources() throws Exception {
- final TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ final TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(whitespaceMockTokenizer(buffer1.toString()));
final TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
final TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
tee1.reset();
@@ -150,7 +151,7 @@ public class TestTeeSinkTokenFilter exte
dogDetector.addAttribute(CheckClearAttributesAttribute.class);
theDetector.addAttribute(CheckClearAttributesAttribute.class);
- final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer2.toString()), MockTokenizer.WHITESPACE, false));
+ final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(whitespaceMockTokenizer(buffer2.toString()));
tee2.addSinkTokenStream(dogDetector);
tee2.addSinkTokenStream(theDetector);
final TokenStream source2 = tee2;
@@ -168,6 +169,12 @@ public class TestTeeSinkTokenFilter exte
lowerCaseTokens[i] = tokens1[i].toLowerCase(Locale.ROOT);
assertTokenStreamContents(lowerCasing, lowerCaseTokens);
}
+
+ private StandardTokenizer standardTokenizer(StringBuilder builder) throws IOException {
+ StandardTokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString()));
+ return tokenizer;
+ }
/**
* Not an explicit test, just useful to print out some info on performance
@@ -182,10 +189,10 @@ public class TestTeeSinkTokenFilter exte
buffer.append(English.intToEnglish(i).toUpperCase(Locale.ROOT)).append(' ');
}
//make sure we produce the same tokens
- TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))));
+ TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)));
TokenStream sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(100));
teeStream.consumeAllTokens();
- TokenStream stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), 100);
+ TokenStream stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)), 100);
CharTermAttribute tfTok = stream.addAttribute(CharTermAttribute.class);
CharTermAttribute sinkTok = sink.addAttribute(CharTermAttribute.class);
for (int i=0; stream.incrementToken(); i++) {
@@ -198,12 +205,12 @@ public class TestTeeSinkTokenFilter exte
int tfPos = 0;
long start = System.currentTimeMillis();
for (int i = 0; i < 20; i++) {
- stream = new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString())));
+ stream = new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer));
PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
while (stream.incrementToken()) {
tfPos += posIncrAtt.getPositionIncrement();
}
- stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), modCounts[j]);
+ stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)), modCounts[j]);
posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
while (stream.incrementToken()) {
tfPos += posIncrAtt.getPositionIncrement();
@@ -215,7 +222,7 @@ public class TestTeeSinkTokenFilter exte
//simulate one field with one sink
start = System.currentTimeMillis();
for (int i = 0; i < 20; i++) {
- teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))));
+ teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)));
sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(modCounts[j]));
PositionIncrementAttribute posIncrAtt = teeStream.getAttribute(PositionIncrementAttribute.class);
while (teeStream.incrementToken()) {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizerTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizerTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizerTest.java Thu Jan 9 13:27:29 2014
@@ -27,7 +27,7 @@ public class TokenRangeSinkTokenizerTest
public void test() throws IOException {
TokenRangeSinkFilter sinkFilter = new TokenRangeSinkFilter(2, 4);
String test = "The quick red fox jumped over the lazy brown dogs";
- TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(whitespaceMockTokenizer(test));
TeeSinkTokenFilter.SinkTokenStream rangeToks = tee.newSinkTokenStream(sinkFilter);
int count = 0;
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java Thu Jan 9 13:27:29 2014
@@ -17,10 +17,8 @@ package org.apache.lucene.analysis.sinks
*/
import java.io.IOException;
-import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -32,7 +30,7 @@ public class TokenTypeSinkTokenizerTest
TokenTypeSinkFilter sinkFilter = new TokenTypeSinkFilter("D");
String test = "The quick red fox jumped over the lazy brown dogs";
- TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
+ TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(whitespaceMockTokenizer(test)));
TeeSinkTokenFilter.SinkTokenStream sink = ttf.newSinkTokenStream(sinkFilter);
boolean seenDogs = false;