You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2013/03/12 21:15:50 UTC
svn commit: r1455698 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/
lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/
lucene/analysis/common/src/java/org/apa...
Author: sarowe
Date: Tue Mar 12 20:15:49 2013
New Revision: 1455698
URL: http://svn.apache.org/r1455698
Log:
LUCENE-4642: Add create(AttributeFactory) to TokenizerFactory and subclasses with ctors taking AttributeFactory, and remove Tokenizer's and subclasses' ctors taking AttributeSource. (merge trunk r1455670)
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue Mar 12 20:15:49 2013
@@ -12,11 +12,13 @@ Changes in backwards compatibility polic
multiple ngrams derived from the same input token. (Walter Underwood
via Mike McCandless)
-
* LUCENE-4822: KeywordTokenFilter is now an abstract class. Subclasses
need to implement #isKeyword() in order to mark terms as keywords.
The existing functionality has been factored out into a new
SetKeywordTokenFilter class. (Simon Willnauer, Uwe Schindler)
+
+* LUCENE-4642: Remove Tokenizer's and subclasses' ctors taking
+ AttributeSource. (Renaud Delbru, Uwe Schindler, Steve Rowe)
New Features
@@ -39,6 +41,10 @@ New Features
determine whether the last token was finished or not, so that a
query "i " will no longer suggest "Isla de Muerta" for example.
(Mike McCandless)
+
+* LUCENE-4642: Add create(AttributeFactory) to TokenizerFactory and
+ subclasses with ctors taking AttributeFactory.
+ (Renaud Delbru, Uwe Schindler, Steve Rowe)
Optimizations
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java Tue Mar 12 20:15:49 2013
@@ -21,7 +21,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer; // javadoc @link
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
@@ -56,20 +55,6 @@ public class ArabicLetterTokenizer exten
}
/**
- * Construct a new ArabicLetterTokenizer using a given {@link AttributeSource}.
- *
- * @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
- * @param source
- * the attribute source to use for this Tokenizer
- * @param in
- * the input to split up into tokens
- */
- public ArabicLetterTokenizer(Version matchVersion, AttributeSource source, Reader in) {
- super(matchVersion, source, in);
- }
-
- /**
* Construct a new ArabicLetterTokenizer using a given
* {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. * @param
* matchVersion Lucene version to match See
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java Tue Mar 12 20:15:49 2013
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.AttributeSource;
/**
* CJKTokenizer is designed for Chinese, Japanese, and Korean languages.
@@ -115,10 +114,6 @@ public final class CJKTokenizer extends
super(in);
}
- public CJKTokenizer(AttributeSource source, Reader in) {
- super(source, in);
- }
-
public CJKTokenizer(AttributeFactory factory, Reader in) {
super(factory, in);
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java Tue Mar 12 20:15:49 2013
@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.standa
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.util.AttributeSource;
/**
@@ -63,10 +62,6 @@ public final class ChineseTokenizer exte
super(in);
}
- public ChineseTokenizer(AttributeSource source, Reader in) {
- super(source, in);
- }
-
public ChineseTokenizer(AttributeFactory factory, Reader in) {
super(factory, in);
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java Tue Mar 12 20:15:49 2013
@@ -23,7 +23,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.util.AttributeSource;
/**
* Emits the entire input as a single token.
@@ -49,14 +48,6 @@ public final class KeywordTokenizer exte
termAtt.resizeBuffer(bufferSize);
}
- public KeywordTokenizer(AttributeSource source, Reader input, int bufferSize) {
- super(source, input);
- if (bufferSize <= 0) {
- throw new IllegalArgumentException("bufferSize must be > 0");
- }
- termAtt.resizeBuffer(bufferSize);
- }
-
public KeywordTokenizer(AttributeFactory factory, Reader input, int bufferSize) {
super(factory, input);
if (bufferSize <= 0) {
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
@@ -37,4 +37,9 @@ public class KeywordTokenizerFactory ext
public KeywordTokenizer create(Reader input) {
return new KeywordTokenizer(input);
}
+
+ @Override
+ public KeywordTokenizer create(AttributeFactory factory, Reader input) {
+ return new KeywordTokenizer(factory, input, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java Tue Mar 12 20:15:49 2013
@@ -21,7 +21,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
@@ -59,20 +58,6 @@ public class LetterTokenizer extends Cha
}
/**
- * Construct a new LetterTokenizer using a given {@link AttributeSource}.
- *
- * @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
- * @param source
- * the attribute source to use for this {@link Tokenizer}
- * @param in
- * the input to split up into tokens
- */
- public LetterTokenizer(Version matchVersion, AttributeSource source, Reader in) {
- super(matchVersion, source, in);
- }
-
- /**
* Construct a new LetterTokenizer using a given
* {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
*
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -45,4 +45,9 @@ public class LetterTokenizerFactory exte
public LetterTokenizer create(Reader input) {
return new LetterTokenizer(luceneMatchVersion, input);
}
+
+ @Override
+ public LetterTokenizer create(AttributeFactory factory, Reader input) {
+ return new LetterTokenizer(luceneMatchVersion, factory, input);
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java Tue Mar 12 20:15:49 2013
@@ -21,7 +21,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
@@ -60,20 +59,6 @@ public final class LowerCaseTokenizer ex
super(matchVersion, in);
}
- /**
- * Construct a new LowerCaseTokenizer using a given {@link AttributeSource}.
- *
- * @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
- * @param source
- * the attribute source to use for this {@link Tokenizer}
- * @param in
- * the input to split up into tokens
- */
- public LowerCaseTokenizer(Version matchVersion, AttributeSource source, Reader in) {
- super(matchVersion, source, in);
- }
-
/**
* Construct a new LowerCaseTokenizer using a given
* {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,10 +17,10 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import org.apache.lucene.analysis.core.LowerCaseTokenizer;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -48,6 +48,11 @@ public class LowerCaseTokenizerFactory e
}
@Override
+ public LowerCaseTokenizer create(AttributeFactory factory, Reader input) {
+ return new LowerCaseTokenizer(luceneMatchVersion, factory, input);
+ }
+
+ @Override
public AbstractAnalysisFactory getMultiTermComponent() {
LowerCaseFilterFactory filt = new LowerCaseFilterFactory();
filt.setLuceneMatchVersion(luceneMatchVersion);
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java Tue Mar 12 20:15:49 2013
@@ -21,7 +21,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
@@ -51,20 +50,6 @@ public final class WhitespaceTokenizer e
}
/**
- * Construct a new WhitespaceTokenizer using a given {@link AttributeSource}.
- *
- * @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
- * @param source
- * the attribute source to use for this {@link Tokenizer}
- * @param in
- * the input to split up into tokens
- */
- public WhitespaceTokenizer(Version matchVersion, AttributeSource source, Reader in) {
- super(matchVersion, source, in);
- }
-
- /**
* Construct a new WhitespaceTokenizer using a given
* {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
*
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
-import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -44,4 +44,9 @@ public class WhitespaceTokenizerFactory
public WhitespaceTokenizer create(Reader input) {
return new WhitespaceTokenizer(luceneMatchVersion,input);
}
+
+ @Override
+ public WhitespaceTokenizer create(AttributeFactory factory, Reader input) {
+ return new WhitespaceTokenizer(luceneMatchVersion, factory, input);
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicTokenizer.java Tue Mar 12 20:15:49 2013
@@ -21,7 +21,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer; // javadocs
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
@@ -35,10 +34,6 @@ public final class IndicTokenizer extend
super(matchVersion, factory, input);
}
- public IndicTokenizer(Version matchVersion, AttributeSource source, Reader input) {
- super(matchVersion, source, input);
- }
-
public IndicTokenizer(Version matchVersion, Reader input) {
super(matchVersion, input);
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java Tue Mar 12 20:15:49 2013
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.util.AttributeSource;
/**
* Tokenizes the input from an edge into n-grams of given size(s).
@@ -96,20 +95,6 @@ public final class EdgeNGramTokenizer ex
/**
* Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range
- *
- * @param source {@link AttributeSource} to use
- * @param input {@link Reader} holding the input to be tokenized
- * @param side the {@link Side} from which to chop off an n-gram
- * @param minGram the smallest n-gram to generate
- * @param maxGram the largest n-gram to generate
- */
- public EdgeNGramTokenizer(AttributeSource source, Reader input, Side side, int minGram, int maxGram) {
- super(source, input);
- init(side, minGram, maxGram);
- }
-
- /**
- * Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range
*
* @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use
* @param input {@link Reader} holding the input to be tokenized
@@ -136,19 +121,6 @@ public final class EdgeNGramTokenizer ex
/**
* Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range
- *
- * @param source {@link AttributeSource} to use
- * @param input {@link Reader} holding the input to be tokenized
- * @param sideLabel the name of the {@link Side} from which to chop off an n-gram
- * @param minGram the smallest n-gram to generate
- * @param maxGram the largest n-gram to generate
- */
- public EdgeNGramTokenizer(AttributeSource source, Reader input, String sideLabel, int minGram, int maxGram) {
- this(source, input, Side.getSide(sideLabel), minGram, maxGram);
- }
-
- /**
- * Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range
*
* @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use
* @param input {@link Reader} holding the input to be tokenized
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.ngram
* limitations under the License.
*/
-import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -59,4 +59,9 @@ public class EdgeNGramTokenizerFactory e
public EdgeNGramTokenizer create(Reader input) {
return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize);
}
+
+ @Override
+ public EdgeNGramTokenizer create(AttributeFactory factory, Reader input) {
+ return new EdgeNGramTokenizer(factory, input, side, minGramSize, maxGramSize);
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java Tue Mar 12 20:15:49 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.analysis.ngram
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.util.AttributeSource;
import java.io.IOException;
import java.io.Reader;
@@ -56,18 +55,6 @@ public final class NGramTokenizer extend
/**
* Creates NGramTokenizer with given min and max n-grams.
- * @param source {@link AttributeSource} to use
- * @param input {@link Reader} holding the input to be tokenized
- * @param minGram the smallest n-gram to generate
- * @param maxGram the largest n-gram to generate
- */
- public NGramTokenizer(AttributeSource source, Reader input, int minGram, int maxGram) {
- super(source, input);
- init(minGram, maxGram);
- }
-
- /**
- * Creates NGramTokenizer with given min and max n-grams.
* @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use
* @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -18,8 +18,8 @@ package org.apache.lucene.analysis.ngram
*/
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.ngram.NGramTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -54,4 +54,9 @@ public class NGramTokenizerFactory exten
public NGramTokenizer create(Reader input) {
return new NGramTokenizer(input, minGramSize, maxGramSize);
}
+
+ @Override
+ public NGramTokenizer create(AttributeFactory factory, Reader input) {
+ return new NGramTokenizer(factory, input, minGramSize, maxGramSize);
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -21,8 +21,6 @@ import java.io.Reader;
import java.util.Map;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.path.PathHierarchyTokenizer;
-import org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
/**
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -22,8 +22,6 @@ import java.io.Reader;
import java.util.Map;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.pattern.PatternTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
/**
@@ -91,7 +89,7 @@ public class PatternTokenizerFactory ext
* Split the input using configured pattern
*/
@Override
- public Tokenizer create(final Reader in) {
+ public PatternTokenizer create(final Reader in) {
try {
return new PatternTokenizer(in, pattern, group);
} catch( IOException ex ) {
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java Tue Mar 12 20:15:49 2013
@@ -22,7 +22,6 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer; // for javadocs
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
@@ -58,20 +57,6 @@ public class RussianLetterTokenizer exte
}
/**
- * Construct a new RussianLetterTokenizer using a given {@link AttributeSource}.
- *
- * @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
- * @param source
- * the attribute source to use for this {@link Tokenizer}
- * @param in
- * the input to split up into tokens
- */
- public RussianLetterTokenizer(Version matchVersion, AttributeSource source, Reader in) {
- super(matchVersion, source, in);
- }
-
- /**
* Construct a new RussianLetterTokenizer using a given
* {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. * @param
* matchVersion Lucene version to match See
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java Tue Mar 12 20:15:49 2013
@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/** A grammar-based tokenizer constructed with JFlex
@@ -104,14 +103,6 @@ public final class ClassicTokenizer exte
}
/**
- * Creates a new ClassicTokenizer with a given {@link AttributeSource}.
- */
- public ClassicTokenizer(Version matchVersion, AttributeSource source, Reader input) {
- super(source, input);
- init(matchVersion);
- }
-
- /**
* Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
*/
public ClassicTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,10 +17,8 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.standard.ClassicTokenizer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -50,9 +48,16 @@ public class ClassicTokenizerFactory ext
}
@Override
- public Tokenizer create(Reader input) {
+ public ClassicTokenizer create(Reader input) {
ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, input);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}
+
+ @Override
+ public ClassicTokenizer create(AttributeFactory factory, Reader input) {
+ ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, factory, input);
+ tokenizer.setMaxTokenLength(maxTokenLength);
+ return tokenizer;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Tue Mar 12 20:15:49 2013
@@ -27,7 +27,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/** A grammar-based tokenizer constructed with JFlex.
@@ -131,14 +130,6 @@ public final class StandardTokenizer ext
}
/**
- * Creates a new StandardTokenizer with a given {@link AttributeSource}.
- */
- public StandardTokenizer(Version matchVersion, AttributeSource source, Reader input) {
- super(source, input);
- init(matchVersion);
- }
-
- /**
* Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
*/
public StandardTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,9 +17,8 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -54,4 +53,12 @@ public class StandardTokenizerFactory ex
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}
+
+ @Override
+ public StandardTokenizer create(AttributeFactory factory, Reader input) {
+ StandardTokenizer tokenizer
+ = new StandardTokenizer(luceneMatchVersion, factory, input);
+ tokenizer.setMaxTokenLength(maxTokenLength);
+ return tokenizer;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java Tue Mar 12 20:15:49 2013
@@ -28,7 +28,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
/**
@@ -110,14 +109,6 @@ public final class UAX29URLEmailTokenize
}
/**
- * Creates a new UAX29URLEmailTokenizer with a given {@link AttributeSource}.
- */
- public UAX29URLEmailTokenizer(Version matchVersion, AttributeSource source, Reader input) {
- super(source, input);
- this.scanner = getScannerFor(matchVersion);
- }
-
- /**
* Creates a new UAX29URLEmailTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
*/
public UAX29URLEmailTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,9 +17,8 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Map;
@@ -54,4 +53,11 @@ public class UAX29URLEmailTokenizerFacto
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}
+
+ @Override
+ public UAX29URLEmailTokenizer create(AttributeFactory factory, Reader input) {
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, factory, input);
+ tokenizer.setMaxTokenLength(maxTokenLength);
+ return tokenizer;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java Tue Mar 12 20:15:49 2013
@@ -23,8 +23,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.CharacterUtils.CharacterBuffer;
@@ -71,7 +69,7 @@ public abstract class CharTokenizer exte
* Creates a new {@link CharTokenizer} instance
*
* @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
+ * Lucene version to match
* @param input
* the input to split up into tokens
*/
@@ -84,23 +82,7 @@ public abstract class CharTokenizer exte
* Creates a new {@link CharTokenizer} instance
*
* @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
- * @param source
- * the attribute source to use for this {@link Tokenizer}
- * @param input
- * the input to split up into tokens
- */
- public CharTokenizer(Version matchVersion, AttributeSource source,
- Reader input) {
- super(source, input);
- charUtils = CharacterUtils.getInstance(matchVersion);
- }
-
- /**
- * Creates a new {@link CharTokenizer} instance
- *
- * @param matchVersion
- * Lucene version to match See {@link <a href="#version">above</a>}
+ * Lucene version to match
* @param factory
* the attribute factory to use for this {@link Tokenizer}
* @param input
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.analysis.util;
*/
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.Set;
@@ -63,4 +64,9 @@ public abstract class TokenizerFactory e
/** Creates a TokenStream of the specified input */
public abstract Tokenizer create(Reader input);
+
+ /** Creates a TokenStream of the specified input using the given AttributeFactory */
+ public Tokenizer create(AttributeFactory factory, Reader input) {
+ throw new UnsupportedOperationException();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java Tue Mar 12 20:15:49 2013
@@ -160,19 +160,6 @@ public final class WikipediaTokenizer ex
init(tokenOutput, untokenizedTypes);
}
- /**
- * Creates a new instance of the {@link org.apache.lucene.analysis.wikipedia.WikipediaTokenizer}. Attaches the
- * <code>input</code> to a the newly created JFlex scanner. Uses the given {@link AttributeSource}.
- *
- * @param input The input
- * @param tokenOutput One of {@link #TOKENS_ONLY}, {@link #UNTOKENIZED_ONLY}, {@link #BOTH}
- */
- public WikipediaTokenizer(AttributeSource source, Reader input, int tokenOutput, Set<String> untokenizedTypes) {
- super(source, input);
- this.scanner = new WikipediaTokenizerImpl(null); // best effort NPE if you dont call reset
- init(tokenOutput, untokenizedTypes);
- }
-
private void init(int tokenOutput, Set<String> untokenizedTypes) {
// TODO: cutover to enum
if (tokenOutput != TOKENS_ONLY &&
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -18,10 +18,10 @@ package org.apache.lucene.analysis.wikip
*/
import java.io.Reader;
+import java.util.Collections;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
/**
* Factory for {@link WikipediaTokenizer}.
@@ -36,7 +36,13 @@ import org.apache.lucene.analysis.wikipe
public class WikipediaTokenizerFactory extends TokenizerFactory {
// TODO: add support for WikipediaTokenizer's advanced options.
@Override
- public Tokenizer create(Reader input) {
+ public WikipediaTokenizer create(Reader input) {
return new WikipediaTokenizer(input);
}
+
+ @Override
+ public WikipediaTokenizer create(AttributeFactory factory, Reader input) {
+ return new WikipediaTokenizer(factory, input, WikipediaTokenizer.TOKENS_ONLY,
+ Collections.<String>emptySet());
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java Tue Mar 12 20:15:49 2013
@@ -146,7 +146,7 @@ public class TestFactories extends BaseT
// some silly classes just so we can use checkRandomData
private TokenizerFactory assertingTokenizer = new TokenizerFactory() {
@Override
- public Tokenizer create(Reader input) {
+ public MockTokenizer create(Reader input) {
return new MockTokenizer(input);
}
};
Modified: lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -25,7 +25,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory; // javadocs
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
@@ -144,7 +143,7 @@ public class ICUTokenizerFactory extends
}
@Override
- public Tokenizer create(Reader input) {
+ public ICUTokenizer create(Reader input) {
assert config != null : "inform must be called first!";
return new ICUTokenizer(input, config);
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -27,8 +27,6 @@ import java.nio.charset.CodingErrorActio
import java.util.Locale;
import java.util.Map;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ja.JapaneseTokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.util.TokenizerFactory;
@@ -89,7 +87,7 @@ public class JapaneseTokenizerFactory ex
}
@Override
- public Tokenizer create(Reader input) {
+ public JapaneseTokenizer create(Reader input) {
return new JapaneseTokenizer(input, userDictionary, discardPunctuation, mode);
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java Tue Mar 12 20:15:49 2013
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.util.AttributeSource;
/**
* Tokenizes input text into sentences.
@@ -52,10 +51,6 @@ public final class SentenceTokenizer ext
super(reader);
}
- public SentenceTokenizer(AttributeSource source, Reader reader) {
- super(source, reader);
- }
-
public SentenceTokenizer(AttributeFactory factory, Reader reader) {
super(factory, reader);
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -19,9 +19,8 @@ package org.apache.lucene.analysis.cn.sm
import java.io.Reader;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.cn.smart.SentenceTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
/**
* Factory for the SmartChineseAnalyzer {@link SentenceTokenizer}
@@ -29,7 +28,12 @@ import org.apache.lucene.analysis.util.T
*/
public class SmartChineseSentenceTokenizerFactory extends TokenizerFactory {
@Override
- public Tokenizer create(Reader input) {
+ public SentenceTokenizer create(Reader input) {
return new SentenceTokenizer(input);
}
+
+ @Override
+ public SentenceTokenizer create(AttributeFactory factory, Reader input) {
+ return new SentenceTokenizer(factory, input);
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,9 +17,7 @@ package org.apache.lucene.analysis.uima;
* limitations under the License.
*/
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer;
import java.io.Reader;
import java.util.HashMap;
@@ -54,7 +52,7 @@ public class UIMAAnnotationsTokenizerFac
}
@Override
- public Tokenizer create(Reader input) {
+ public UIMAAnnotationsTokenizer create(Reader input) {
return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters, input);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.analysis.uima;
* limitations under the License.
*/
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
import java.io.Reader;
@@ -55,7 +54,7 @@ public class UIMATypeAwareAnnotationsTok
}
@Override
- public Tokenizer create(Reader input) {
+ public UIMATypeAwareAnnotationsTokenizer create(Reader input) {
return new UIMATypeAwareAnnotationsTokenizer(descriptorPath, tokenType, featurePath, configurationParameters, input);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java Tue Mar 12 20:15:49 2013
@@ -47,13 +47,6 @@ public abstract class Tokenizer extends
this.input = input;
}
- /** Construct a token stream processing the given input using the given AttributeSource. */
- protected Tokenizer(AttributeSource source, Reader input) {
- super(source);
- assert input != null: "input must not be null";
- this.input = input;
- }
-
/**
* {@inheritDoc}
* <p>
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -22,12 +22,15 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.DateField;
import static org.apache.solr.schema.TrieField.TrieTypes;
import java.io.IOException;
import java.io.Reader;
+import java.util.Iterator;
/**
* Tokenizer for trie fields. It uses NumericTokenStream to create multiple trie encoded string per number.
@@ -71,9 +74,18 @@ final class TrieTokenizer extends Tokeni
return new NumericTokenStream(precisionStep);
}
- public TrieTokenizer(Reader input, TrieTypes type, NumericTokenStream ts) {
- // must share the attribute source with the NumericTokenStream we delegate to
- super(ts, input);
+ public TrieTokenizer(Reader input, TrieTypes type, final NumericTokenStream ts) {
+ // Häckidy-Hick-Hack: must share the attributes with the NumericTokenStream we delegate to, so we create a fake factory:
+ super(new AttributeFactory() {
+ @Override
+ public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
+ return (AttributeImpl) ts.addAttribute(attClass);
+ }
+ }, input);
+ // add all attributes:
+ for (Iterator<Class<? extends Attribute>> it = ts.getAttributeClassesIterator(); it.hasNext();) {
+ addAttribute(it.next());
+ }
this.type = type;
this.ts = ts;
// dates tend to be longer, especially when math is involved
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java Tue Mar 12 20:15:49 2013
@@ -20,7 +20,6 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.util.Map;
-import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockCharFilter;
import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -41,7 +40,7 @@ public class MockCharFilterFactory exten
}
@Override
- public CharFilter create(Reader input) {
+ public MockCharFilter create(Reader input) {
return new MockCharFilter(input, remainder);
}
}
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java?rev=1455698&r1=1455697&r2=1455698&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java Tue Mar 12 20:15:49 2013
@@ -21,7 +21,6 @@ import java.io.Reader;
import java.util.Map;
import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
@@ -55,7 +54,7 @@ public class MockTokenizerFactory extend
@Override
- public Tokenizer create(Reader input) {
+ public MockTokenizer create(Reader input) {
MockTokenizer t = new MockTokenizer(input, pattern, false);
t.setEnableChecks(enableChecks);
return t;