You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/01/09 14:27:36 UTC
svn commit: r1556801 [10/10] - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/
lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/
lucene/analysis/common/src/java/org/apache/lucene/analysis/br/ lu...
Modified: lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (original)
+++ lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java Thu Jan 9 13:27:29 2014
@@ -118,8 +118,8 @@ public abstract class QueryParserTestBas
/** Filters MockTokenizer with StopFilter. */
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer));
}
}
@@ -250,8 +250,8 @@ public abstract class QueryParserTestBas
protected static class SimpleCJKTokenizer extends Tokenizer {
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- public SimpleCJKTokenizer(Reader input) {
- super(input);
+ public SimpleCJKTokenizer() {
+ super();
}
@Override
@@ -267,8 +267,8 @@ public abstract class QueryParserTestBas
private class SimpleCJKAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new SimpleCJKTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new SimpleCJKTokenizer());
}
}
@@ -403,8 +403,8 @@ public abstract class QueryParserTestBas
// +,-,! should be directly adjacent to operand (i.e. not separated by whitespace) to be treated as an operator
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
}
};
assertQueryEquals("a - b", a, "a - b");
@@ -1133,8 +1133,8 @@ public abstract class QueryParserTestBas
super();
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer( MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new MockSynonymFilter(tokenizer));
}
}
@@ -1145,8 +1145,8 @@ public abstract class QueryParserTestBas
super();
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
}
}
@@ -1176,8 +1176,8 @@ public abstract class QueryParserTestBas
}
private class MockCollationAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new MockCollationFilter(tokenizer));
}
}
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Thu Jan 9 13:27:29 2014
@@ -397,8 +397,8 @@ public class AnalyzingInfixSuggesterTest
final CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "a");
Analyzer indexAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokens = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokens = new MockTokenizer();
return new TokenStreamComponents(tokens,
new StopFilter(TEST_VERSION_CURRENT, tokens, stopWords));
}
@@ -406,8 +406,8 @@ public class AnalyzingInfixSuggesterTest
Analyzer queryAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokens = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokens = new MockTokenizer();
return new TokenStreamComponents(tokens,
new SuggestStopFilter(tokens, stopWords));
}
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Thu Jan 9 13:27:29 2014
@@ -273,8 +273,8 @@ public class AnalyzingSuggesterTest exte
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -346,8 +346,8 @@ public class AnalyzingSuggesterTest exte
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -424,8 +424,8 @@ public class AnalyzingSuggesterTest exte
private final Analyzer getUnusualAnalyzer() {
return new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -631,8 +631,8 @@ public class AnalyzingSuggesterTest exte
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(factory, reader, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(factory, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
tokenizer.setEnableChecks(true);
TokenStream next;
if (numStopChars != 0) {
@@ -948,8 +948,8 @@ public class AnalyzingSuggesterTest exte
public void testDupSurfaceFormsMissingResults() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -1007,8 +1007,8 @@ public class AnalyzingSuggesterTest exte
public void testDupSurfaceFormsMissingResults2() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -1077,8 +1077,8 @@ public class AnalyzingSuggesterTest exte
public void test0ByteKeys() throws Exception {
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -1147,8 +1147,8 @@ public class AnalyzingSuggesterTest exte
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@Override
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java Thu Jan 9 13:27:29 2014
@@ -227,8 +227,8 @@ public class FuzzySuggesterTest extends
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -308,8 +308,8 @@ public class FuzzySuggesterTest extends
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -382,8 +382,8 @@ public class FuzzySuggesterTest extends
private final Analyzer getUnusualAnalyzer() {
return new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -579,8 +579,8 @@ public class FuzzySuggesterTest extends
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
tokenizer.setEnableChecks(true);
TokenStream next;
if (numStopChars != 0) {
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java Thu Jan 9 13:27:29 2014
@@ -231,8 +231,8 @@ public class TestFreeTextSuggester exten
// Just deletes "of"
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer();
CharArraySet stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "of");
return new TokenStreamComponents(tokenizer, new StopFilter(TEST_VERSION_CURRENT, tokenizer, stopSet));
}
@@ -259,8 +259,8 @@ public class TestFreeTextSuggester exten
// Just deletes "of"
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer();
CharArraySet stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "of");
return new TokenStreamComponents(tokenizer, new StopFilter(TEST_VERSION_CURRENT, tokenizer, stopSet));
}
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestSuggestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestSuggestStopFilter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestSuggestStopFilter.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestSuggestStopFilter.java Thu Jan 9 13:27:29 2014
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -29,7 +30,8 @@ public class TestSuggestStopFilter exten
public void testEndNotStopWord() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to");
- TokenStream stream = new MockTokenizer(new StringReader("go to"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
assertTokenStreamContents(filter,
new String[] {"go", "to"},
@@ -46,9 +48,9 @@ public class TestSuggestStopFilter exten
public void testEndIsStopWord() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to");
- TokenStream stream = new MockTokenizer(new StringReader("go to "));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to "));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
-
filter = new SuggestStopFilter(stream, stopWords);
assertTokenStreamContents(filter,
new String[] {"go"},
@@ -65,7 +67,8 @@ public class TestSuggestStopFilter exten
public void testMidStopWord() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to");
- TokenStream stream = new MockTokenizer(new StringReader("go to school"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to school"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
@@ -84,7 +87,8 @@ public class TestSuggestStopFilter exten
public void testMultipleStopWords() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
- TokenStream stream = new MockTokenizer(new StringReader("go to a the school"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to a the school"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
@@ -103,7 +107,8 @@ public class TestSuggestStopFilter exten
public void testMultipleStopWordsEnd() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
- TokenStream stream = new MockTokenizer(new StringReader("go to a the"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to a the"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
@@ -122,7 +127,8 @@ public class TestSuggestStopFilter exten
public void testMultipleStopWordsEnd2() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
- TokenStream stream = new MockTokenizer(new StringReader("go to a the "));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to a the "));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Thu Jan 9 13:27:29 2014
@@ -972,4 +972,29 @@ public abstract class BaseTokenStreamTes
}
return ret;
}
+
+ protected static MockTokenizer whitespaceMockTokenizer(Reader input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(input);
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer whitespaceMockTokenizer(String input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(new StringReader(input));
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer keywordMockTokenizer(Reader input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ mockTokenizer.setReader(input);
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer keywordMockTokenizer(String input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ mockTokenizer.setReader(new StringReader(input));
+ return mockTokenizer;
+ }
+
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java Thu Jan 9 13:27:29 2014
@@ -87,8 +87,8 @@ public final class MockAnalyzer extends
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase, maxTokenLength);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(runAutomaton, lowerCase, maxTokenLength);
tokenizer.setEnableChecks(enableChecks);
MockTokenFilter filt = new MockTokenFilter(tokenizer, filter);
return new TokenStreamComponents(tokenizer, maybePayload(filt, fieldName));
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java Thu Jan 9 13:27:29 2014
@@ -17,8 +17,6 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
-import java.io.Reader;
-
/**
* Analyzer for testing that encodes terms as UTF-16 bytes.
*/
@@ -26,8 +24,8 @@ public class MockBytesAnalyzer extends A
private final MockBytesAttributeFactory factory = new MockBytesAttributeFactory();
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(factory, reader, MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer(factory, MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
return new TokenStreamComponents(t);
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java Thu Jan 9 13:27:29 2014
@@ -34,8 +34,8 @@ import java.io.Reader;
public final class MockPayloadAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer( MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new MockPayloadFilter(result, fieldName));
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java Thu Jan 9 13:27:29 2014
@@ -89,35 +89,34 @@ public class MockTokenizer extends Token
// evil: but we don't change the behavior with this random, we only switch up how we read
private final Random random = new Random(RandomizedContext.current().getRandom().nextLong());
- public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
- super(factory, input);
+ public MockTokenizer(AttributeFactory factory, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
+ super(factory);
this.runAutomaton = runAutomaton;
this.lowerCase = lowerCase;
this.state = runAutomaton.getInitialState();
- this.streamState = State.SETREADER;
this.maxTokenLength = maxTokenLength;
}
- public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, runAutomaton, lowerCase, maxTokenLength);
+ public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, runAutomaton, lowerCase, maxTokenLength);
}
- public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
- this(input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
+ public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
}
- /** Calls {@link #MockTokenizer(Reader, CharacterRunAutomaton, boolean) MockTokenizer(Reader, WHITESPACE, true)} */
- public MockTokenizer(Reader input) {
- this(input, WHITESPACE, true);
+ /** Calls {@link #MockTokenizer(CharacterRunAutomaton, boolean) MockTokenizer(Reader, WHITESPACE, true)} */
+ public MockTokenizer() {
+ this(WHITESPACE, true);
}
- public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
- this(factory, input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
+ public MockTokenizer(AttributeFactory factory, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(factory, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
}
- /** Calls {@link #MockTokenizer(org.apache.lucene.util.AttributeSource.AttributeFactory,Reader,CharacterRunAutomaton,boolean)
+ /** Calls {@link #MockTokenizer(org.apache.lucene.util.AttributeSource.AttributeFactory,CharacterRunAutomaton,boolean)
* MockTokenizer(AttributeFactory, Reader, WHITESPACE, true)} */
- public MockTokenizer(AttributeFactory factory, Reader input) {
- this(input, WHITESPACE, true);
+ public MockTokenizer(AttributeFactory factory) {
+ this(factory, WHITESPACE, true);
}
@Override
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java Thu Jan 9 13:27:29 2014
@@ -60,8 +60,8 @@ public final class TokenizerChain extend
}
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
- Tokenizer tk = tokenizer.create( aReader );
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tk = tokenizer.create();
TokenStream ts = tk;
for (TokenFilterFactory filter : filters) {
ts = filter.create(ts);
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java Thu Jan 9 13:27:29 2014
@@ -19,6 +19,7 @@ package org.apache.solr.handler;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -110,7 +111,12 @@ public abstract class AnalysisRequestHan
}
}
- TokenStream tokenStream = tfac.create(tokenizerChain.initReader(null, new StringReader(value)));
+ TokenStream tokenStream = tfac.create();
+ try {
+ ((Tokenizer)tokenStream).setReader(tokenizerChain.initReader(null, new StringReader(value)));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
List<AttributeSource> tokens = analyzeTokenStream(tokenStream);
namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java Thu Jan 9 13:27:29 2014
@@ -67,8 +67,8 @@ public class BoolField extends Primitive
protected final static Analyzer boolAnalyzer = new SolrAnalyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new Tokenizer(reader) {
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new Tokenizer() {
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
boolean done = false;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java Thu Jan 9 13:27:29 2014
@@ -439,8 +439,8 @@ public abstract class FieldType extends
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer ts = new Tokenizer(reader) {
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer ts = new Tokenizer() {
final char[] cbuf = new char[maxChars];
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java Thu Jan 9 13:27:29 2014
@@ -86,8 +86,8 @@ public class PreAnalyzedField extends Fi
return new SolrAnalyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new PreAnalyzedTokenizer(reader, parser));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new PreAnalyzedTokenizer(parser));
}
};
@@ -198,7 +198,8 @@ public class PreAnalyzedField extends Fi
if (val == null || val.trim().length() == 0) {
return null;
}
- PreAnalyzedTokenizer parse = new PreAnalyzedTokenizer(new StringReader(val), parser);
+ PreAnalyzedTokenizer parse = new PreAnalyzedTokenizer(parser);
+ parse.setReader(new StringReader(val));
parse.reset(); // consume
org.apache.lucene.document.FieldType type = createFieldType(field);
if (type == null) {
@@ -251,12 +252,8 @@ public class PreAnalyzedField extends Fi
private String stringValue = null;
private byte[] binaryValue = null;
private PreAnalyzedParser parser;
- private Reader lastReader;
- private Reader input; // hides original input since we replay saved states (and dont reuse)
- public PreAnalyzedTokenizer(Reader reader, PreAnalyzedParser parser) {
- super(reader);
- this.input = reader;
+ public PreAnalyzedTokenizer(PreAnalyzedParser parser) {
this.parser = parser;
}
@@ -271,7 +268,7 @@ public class PreAnalyzedField extends Fi
public byte[] getBinaryValue() {
return binaryValue;
}
-
+
@Override
public final boolean incrementToken() {
// lazy init the iterator
@@ -291,8 +288,8 @@ public class PreAnalyzedField extends Fi
@Override
public final void reset() throws IOException {
// NOTE: this acts like rewind if you call it again
- if (input != lastReader) {
- lastReader = input;
+ if (it == null) {
+ super.reset();
cachedStates.clear();
stringValue = null;
binaryValue = null;
@@ -307,12 +304,6 @@ public class PreAnalyzedField extends Fi
}
it = cachedStates.iterator();
}
-
- @Override
- public void close() throws IOException {
- super.close();
- lastReader = null; // just a ref, null for gc
- }
}
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java Thu Jan 9 13:27:29 2014
@@ -260,8 +260,8 @@ public class LegacyHTMLStripCharFilterTe
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -66,7 +66,7 @@ public class TestReversedWildcardFilterF
String text = "simple text";
args.put("withOriginal", "true");
ReversedWildcardFilterFactory factory = new ReversedWildcardFilterFactory(args);
- TokenStream input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
+ TokenStream input = factory.create(whitespaceMockTokenizer(text));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "simple", "\u0001txet", "text" },
new int[] { 1, 0, 1, 0 });
@@ -74,7 +74,7 @@ public class TestReversedWildcardFilterF
// now without original tokens
args.put("withOriginal", "false");
factory = new ReversedWildcardFilterFactory(args);
- input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
+ input = factory.create(whitespaceMockTokenizer(text));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "\u0001txet" },
new int[] { 1, 1 });
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java Thu Jan 9 13:27:29 2014
@@ -211,13 +211,11 @@ public class TestWordDelimiterFilterFact
WordDelimiterFilterFactory factoryDefault = new WordDelimiterFilterFactory(args);
factoryDefault.inform(loader);
- TokenStream ts = factoryDefault.create(
- new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
+ TokenStream ts = factoryDefault.create(whitespaceMockTokenizer(testText));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "5", "400", "00", "540000", "at", "25", "interest", "rate", "interestrate" });
- ts = factoryDefault.create(
- new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
+ ts = factoryDefault.create(whitespaceMockTokenizer("foo\u200Dbar"));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo", "bar", "foobar" });
@@ -235,14 +233,12 @@ public class TestWordDelimiterFilterFact
WordDelimiterFilterFactory factoryCustom = new WordDelimiterFilterFactory(args);
factoryCustom.inform(loader);
- ts = factoryCustom.create(
- new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
+ ts = factoryCustom.create(whitespaceMockTokenizer(testText));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "$5,400.00", "at", "25%", "interest", "rate", "interestrate" });
/* test custom behavior with a char > 0x7F, because we had to make a larger byte[] */
- ts = factoryCustom.create(
- new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
+ ts = factoryCustom.create(whitespaceMockTokenizer("foo\u200Dbar"));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo\u200Dbar" });
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java Thu Jan 9 13:27:29 2014
@@ -48,8 +48,8 @@ public class TestSuggestSpellingConverte
// lowercases, removes field names, other syntax, collapses runs of whitespace, etc.
converter.setAnalyzer(new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
TokenStream filter = new PatternReplaceFilter(tokenizer,
Pattern.compile("([^\\p{L}\\p{M}\\p{N}\\p{Cs}]*[\\p{L}\\p{M}\\p{N}\\p{Cs}\\_]+:)|([^\\p{L}\\p{M}\\p{N}\\p{Cs}])+"), " ", true);
filter = new LowerCaseFilter(TEST_VERSION_CURRENT, filter);
Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java Thu Jan 9 13:27:29 2014
@@ -21,6 +21,8 @@ import static com.google.common.base.Pre
import java.io.File;
import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
@@ -41,6 +43,7 @@ import javax.xml.xpath.XPathExpressionEx
import org.apache.commons.codec.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -174,6 +177,19 @@ public abstract class SolrTestCaseJ4 ext
}
}
+
+ protected static MockTokenizer whitespaceMockTokenizer(Reader input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(input);
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer whitespaceMockTokenizer(String input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(new StringReader(input));
+ return mockTokenizer;
+ }
+
/**
* Call this from @BeforeClass to set up the test harness and update handler with no cores.
*
@@ -1770,4 +1786,5 @@ public abstract class SolrTestCaseJ4 ext
}
}
+
}
Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java?rev=1556801&r1=1556800&r2=1556801&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java Thu Jan 9 13:27:29 2014
@@ -52,8 +52,8 @@ public class MockTokenizerFactory extend
}
@Override
- public MockTokenizer create(AttributeFactory factory, Reader input) {
- MockTokenizer t = new MockTokenizer(factory, input, pattern, false);
+ public MockTokenizer create(AttributeFactory factory) {
+ MockTokenizer t = new MockTokenizer(factory, pattern, false);
t.setEnableChecks(enableChecks);
return t;
}