You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/02/10 21:13:58 UTC
svn commit: r1242908 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/
lucene/contrib/queryparser/src/te...
Author: rmuir
Date: Fri Feb 10 20:13:57 2012
New Revision: 1242908
URL: http://svn.apache.org/viewvc?rev=1242908&view=rev
Log:
LUCENE-3766: Deprecate Tokenizer's default ctor
Added:
lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java
- copied unchanged from r1242890, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java
lucene/dev/branches/branch_3x/lucene/test-framework/src/java/org/apache/lucene/analysis/EmptyTokenizer.java
- copied unchanged from r1242890, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/EmptyTokenizer.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/core/ (props changed)
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Fri Feb 10 20:13:57 2012
@@ -100,6 +100,11 @@ API Changes
Deprecated StandardAnalyzer and ClassicAnalyzer ctors that take File,
please use the Reader ctor instead. (Robert Muir)
+* LUCENE-3766: Deprecate no-arg ctors of Tokenizer. Tokenizers are
+ TokenStreams with Readers: tokenizers with null Readers will not be
+ supported in Lucene 4.0, just use a TokenStream.
+ (Mike McCandless, Robert Muir)
+
New Features
* LUCENE-3593: Added a FieldValueFilter that accepts all documents that either
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java Fri Feb 10 20:13:57 2012
@@ -18,9 +18,7 @@ package org.apache.lucene.search.highlig
*/
import java.io.IOException;
-import java.io.Reader;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -49,15 +47,7 @@ import org.apache.lucene.util.LuceneTest
public class TokenSourcesTest extends LuceneTestCase {
private static final String FIELD = "text";
- private static final class OverlapAnalyzer extends Analyzer {
-
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new TokenStreamOverlap();
- }
- }
-
- private static final class TokenStreamOverlap extends TokenStream {
+ private static final class OverlappingTokenStream extends TokenStream {
private Token[] tokens;
private int i = -1;
@@ -66,10 +56,6 @@ public class TokenSourcesTest extends Lu
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
- public TokenStreamOverlap() {
- reset();
- }
-
@Override
public boolean incrementToken() throws IOException {
this.i++;
@@ -104,10 +90,10 @@ public class TokenSourcesTest extends Lu
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, null));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
+ document.add(new Field(FIELD, new OverlappingTokenStream(),
TermVector.WITH_OFFSETS));
indexWriter.addDocument(document);
} finally {
@@ -150,10 +136,10 @@ public class TokenSourcesTest extends Lu
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, null));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
+ document.add(new Field(FIELD, new OverlappingTokenStream(),
TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(document);
} finally {
@@ -196,10 +182,10 @@ public class TokenSourcesTest extends Lu
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, null));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
+ document.add(new Field(FIELD, new OverlappingTokenStream(),
TermVector.WITH_OFFSETS));
indexWriter.addDocument(document);
} finally {
@@ -243,10 +229,10 @@ public class TokenSourcesTest extends Lu
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, null));
try {
final Document document = new Document();
- document.add(new Field(FIELD, new TokenStreamOverlap(),
+ document.add(new Field(FIELD, new OverlappingTokenStream(),
TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(document);
} finally {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java Fri Feb 10 20:13:57 2012
@@ -25,6 +25,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -300,7 +301,7 @@ public class IndexTimeSynonymTest extend
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
+ Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, reader) {
final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
int p = 0;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Fri Feb 10 20:13:57 2012
@@ -1221,11 +1221,15 @@ public class TestQPHelper extends Lucene
super.tearDown();
}
- private class CannedTokenStream extends Tokenizer {
+ private class CannedTokenizer extends Tokenizer {
private int upto = 0;
private final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
+ CannedTokenizer(Reader reader) {
+ super(reader);
+ }
+
@Override
public boolean incrementToken() {
clearAttributes();
@@ -1259,7 +1263,7 @@ public class TestQPHelper extends Lucene
private class CannedAnalyzer extends Analyzer {
@Override
public TokenStream tokenStream(String ignored, Reader alsoIgnored) {
- return new CannedTokenStream();
+ return new CannedTokenizer(alsoIgnored);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java Fri Feb 10 20:13:57 2012
@@ -34,7 +34,10 @@ public abstract class Tokenizer extends
/** The text source for this Tokenizer. */
protected Reader input;
- /** Construct a tokenizer with null input. */
+ /** Construct a tokenizer with null input.
+ * @deprecated use {@link #Tokenizer(Reader)} instead.
+ */
+ @Deprecated
protected Tokenizer() {}
/** Construct a token stream processing the given input. */
@@ -42,7 +45,10 @@ public abstract class Tokenizer extends
this.input = CharReader.get(input);
}
- /** Construct a tokenizer with null input using the given AttributeFactory. */
+ /** Construct a tokenizer with null input using the given AttributeFactory.
+ * @deprecated use {@link #Tokenizer(AttributeSource.AttributeFactory, Reader)} instead.
+ */
+ @Deprecated
protected Tokenizer(AttributeFactory factory) {
super(factory);
}
@@ -53,7 +59,10 @@ public abstract class Tokenizer extends
this.input = CharReader.get(input);
}
- /** Construct a token stream processing the given input using the given AttributeSource. */
+ /** Construct a token stream processing the given input using the given AttributeSource.
+ * @deprecated use {@link #Tokenizer(AttributeSource, Reader)} instead.
+ */
+ @Deprecated
protected Tokenizer(AttributeSource source) {
super(source);
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java Fri Feb 10 20:13:57 2012
@@ -114,35 +114,34 @@ public final class ClassicTokenizer exte
* See http://issues.apache.org/jira/browse/LUCENE-1068
*/
public ClassicTokenizer(Version matchVersion, Reader input) {
- super();
- init(input, matchVersion);
+ super(input);
+ init(matchVersion);
}
/**
* Creates a new ClassicTokenizer with a given {@link AttributeSource}.
*/
public ClassicTokenizer(Version matchVersion, AttributeSource source, Reader input) {
- super(source);
- init(input, matchVersion);
+ super(source, input);
+ init(matchVersion);
}
/**
* Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
*/
public ClassicTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
- super(factory);
- init(input, matchVersion);
+ super(factory, input);
+ init(matchVersion);
}
- private final void init(Reader input, Version matchVersion) {
+ private final void init(Version matchVersion) {
this.scanner = new ClassicTokenizerImpl(input);
if (matchVersion.onOrAfter(Version.LUCENE_24)) {
replaceInvalidAcronym = true;
} else {
replaceInvalidAcronym = false;
- }
- this.input = input;
+ }
}
// this tokenizer generates three attributes:
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Fri Feb 10 20:13:57 2012
@@ -130,27 +130,27 @@ public final class StandardTokenizer ext
* See http://issues.apache.org/jira/browse/LUCENE-1068
*/
public StandardTokenizer(Version matchVersion, Reader input) {
- super();
- init(input, matchVersion);
+ super(input);
+ init(matchVersion);
}
/**
* Creates a new StandardTokenizer with a given {@link AttributeSource}.
*/
public StandardTokenizer(Version matchVersion, AttributeSource source, Reader input) {
- super(source);
- init(input, matchVersion);
+ super(source, input);
+ init(matchVersion);
}
/**
* Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
*/
public StandardTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
- super(factory);
- init(input, matchVersion);
+ super(factory, input);
+ init(matchVersion);
}
- private final void init(Reader input, Version matchVersion) {
+ private final void init(Version matchVersion) {
if (matchVersion.onOrAfter(Version.LUCENE_34)) {
this.scanner = new StandardTokenizerImpl(input);
} else if (matchVersion.onOrAfter(Version.LUCENE_31)) {
@@ -162,8 +162,7 @@ public final class StandardTokenizer ext
replaceInvalidAcronym = true;
} else {
replaceInvalidAcronym = false;
- }
- this.input = input;
+ }
}
// this tokenizer generates three attributes:
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java Fri Feb 10 20:13:57 2012
@@ -127,7 +127,7 @@ public class AttributeSource {
private final Map<Class<? extends AttributeImpl>, AttributeImpl> attributeImpls;
private final State[] currentState;
- private AttributeFactory factory;
+ private final AttributeFactory factory;
/**
* An AttributeSource using the default attribute factory {@link AttributeSource.AttributeFactory#DEFAULT_ATTRIBUTE_FACTORY}.
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Feb 10 20:13:57 2012
@@ -1731,6 +1731,7 @@ public class TestIndexWriter extends Luc
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
public StringSplitTokenizer(Reader r) {
+ super(r);
try {
reset(r);
} catch (IOException e) {
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java Fri Feb 10 20:13:57 2012
@@ -18,12 +18,8 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.io.Reader;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -40,10 +36,9 @@ public class TestSameTokenSamePosition e
*/
public void test() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter riw = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new BugReproAnalyzer()));
+ RandomIndexWriter riw = new RandomIndexWriter(random, dir);
Document doc = new Document();
- doc.add(new Field("eng", "Six drunken" /*This shouldn't matter. */,
- Field.Store.YES, Field.Index.ANALYZED));
+ doc.add(new Field("eng", new BugReproTokenStream()));
riw.addDocument(doc);
riw.close();
dir.close();
@@ -54,11 +49,10 @@ public class TestSameTokenSamePosition e
*/
public void testMoreDocs() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter riw = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new BugReproAnalyzer()));
- Document doc = new Document();
- doc.add(new Field("eng", "Six drunken" /*This shouldn't matter. */,
- Field.Store.YES, Field.Index.ANALYZED));
+ RandomIndexWriter riw = new RandomIndexWriter(random, dir);
for (int i = 0; i < 100; i++) {
+ Document doc = new Document();
+ doc.add(new Field("eng", new BugReproTokenStream()));
riw.addDocument(doc);
}
riw.close();
@@ -66,14 +60,7 @@ public class TestSameTokenSamePosition e
}
}
-final class BugReproAnalyzer extends Analyzer{
- @Override
- public TokenStream tokenStream(String arg0, Reader arg1) {
- return new BugReproAnalyzerTokenizer();
- }
-}
-
-final class BugReproAnalyzerTokenizer extends Tokenizer {
+final class BugReproTokenStream extends TokenStream {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java Fri Feb 10 20:13:57 2012
@@ -121,14 +121,15 @@ public class TestTermVectorsReader exten
super.tearDown();
}
- private class MyTokenStream extends TokenStream {
+ private class MyTokenizer extends Tokenizer {
private int tokenUpto;
private final CharTermAttribute termAtt;
private final PositionIncrementAttribute posIncrAtt;
private final OffsetAttribute offsetAtt;
- public MyTokenStream() {
+ public MyTokenizer(Reader reader) {
+ super(reader);
termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
@@ -162,7 +163,7 @@ public class TestTermVectorsReader exten
private class MyAnalyzer extends Analyzer {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MyTokenStream();
+ return new MyTokenizer(reader);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java Fri Feb 10 20:13:57 2012
@@ -40,7 +40,8 @@ class RepeatingTokenStream extends Token
CharTermAttribute termAtt;
String value;
- public RepeatingTokenStream(String val, Random random, float percentDocs, int maxTF) {
+ public RepeatingTokenStream(Reader reader, String val, Random random, float percentDocs, int maxTF) {
+ super(reader);
this.value = val;
this.random = random;
this.percentDocs = percentDocs;
@@ -74,12 +75,11 @@ class RepeatingTokenStream extends Token
public class TestTermdocPerf extends LuceneTestCase {
void addDocs(final Random random, Directory dir, final int ndocs, String field, final String val, final int maxTF, final float percentDocs) throws IOException {
- final RepeatingTokenStream ts = new RepeatingTokenStream(val, random, percentDocs, maxTF);
Analyzer analyzer = new Analyzer() {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return ts;
+ return new RepeatingTokenStream(reader, val, random, percentDocs, maxTF);
}
};
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Fri Feb 10 20:13:57 2012
@@ -353,7 +353,7 @@ public class TestMultiPhraseQuery extend
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new CannedTokenizer(tokens);
+ return new CannedTokenizer(reader, tokens);
}
}
@@ -364,7 +364,8 @@ public class TestMultiPhraseQuery extend
private final TermAttribute termAtt = addAttribute(TermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- public CannedTokenizer(TokenAndPos[] tokens) {
+ public CannedTokenizer(Reader reader, TokenAndPos[] tokens) {
+ super(reader);
this.tokens = tokens;
}
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1242908&r1=1242907&r2=1242908&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Fri Feb 10 20:13:57 2012
@@ -67,7 +67,7 @@ final class TrieTokenizer extends Tokeni
public TrieTokenizer(Reader input, TrieTypes type, int precisionStep, NumericTokenStream ts) {
// must share the attribute source with the NumericTokenStream we delegate to
- super(ts);
+ super(ts, input);
this.type = type;
this.precisionStep = precisionStep;
this.ts = ts;